@open-mercato/core 0.6.8-develop.7100.1.fbf66fca35 → 0.7.0

package/src/modules/query_index/lib/engine.ts

@@ -1,4 +1,4 @@
-import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, ListCountCapWarning, FilterOp, Filter, QueryCustomFieldSource, PartialIndexWarning, QueryExtensionsConfig, Sort } from '@open-mercato/shared/lib/query/types'
+import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, FilterOp, Filter, QueryCustomFieldSource, PartialIndexWarning, QueryExtensionsConfig, Sort } from '@open-mercato/shared/lib/query/types'
 import { SortDir } from '@open-mercato/shared/lib/query/types'
 import type { EntityId } from '@open-mercato/shared/modules/entities'
 import type { EntityManager } from '@mikro-orm/postgresql'
@@ -20,8 +20,6 @@ import {
   type ResolvedJoin,
 } from '@open-mercato/shared/lib/query/join-utils'
 import { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'
-import { isEncryptedLikeField, resolveEncryptedLikeFieldSet } from '@open-mercato/shared/lib/query/engine'
-import { isTenantDataEncryptionEnabled } from '@open-mercato/shared/lib/encryption/toggles'
 import {
   createSearchTokenAvailability,
   isSearchFilterOp,
@@ -34,7 +32,6 @@ import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from '@open-mercato/shared/lib/query/query-extension-runner'
 import { warnOnCiphertextLikeFallback } from '@open-mercato/shared/lib/query/ciphertext-search-warning'
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from '@open-mercato/shared/lib/query/encrypted-sort'
-import { resolveListCountCap } from '@open-mercato/shared/lib/query/count-cap'
 import { mapWithConcurrency } from '@open-mercato/shared/lib/query/bounded-decrypt'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
@@ -151,25 +148,13 @@ type SearchRuntime = {
   organizationScope?: { ids: string[]; includeNull: boolean } | null
   tenantId?: string | null
   searchSources?: SearchTokenSource[]
-  /**
-   * Base-column fields whose stored value is ciphertext, so a like/ilike on them can only be
-   * answered via search tokens. A plaintext column keeps exact SQL ILIKE instead: the token
-   * rewrite is approximate -- it splits on non-alphanumerics and drops tokens shorter than
-   * minTokenLength, so a document-number search like "ZK 1/2026" degrades to the tokens
-   * {202, 2026} and matches every record from that year, and an all-short term like "ZK"
-   * produces no tokens and silently drops the predicate. `null`/absent = the encryption
-   * service could not answer (or a caller predates this field); keep the old rewrite then,
-   * because guessing "plaintext" would turn encrypted-column search into an
-   * ILIKE-on-ciphertext that matches nothing.
-   */
-  encryptedFields?: Set<string> | null
   /** Per-`query()` alias minter for `search_tokens` subqueries (see #2738). */
   mintAlias: () => string
 }
 
 type EncryptionResolver = () => {
   decryptEntityPayload?: (entityId: EntityId, payload: Record<string, unknown>, tenantId?: string | null, organizationId?: string | null) => Promise<Record<string, unknown>>
-  getEncryptedFieldNames?: (entityId: EntityId, tenantId?: string | null, organizationId?: string | null, options?: { ignoreRuntimeHealth?: boolean }) => Promise<readonly string[]>
+  getEncryptedFieldNames?: (entityId: EntityId, tenantId?: string | null, organizationId?: string | null) => Promise<readonly string[]>
   isEnabled?: () => boolean
 } | null
 
@@ -511,51 +496,6 @@ export class HybridQueryEngine implements QueryEngine {
         ? await this.searchAvailability().anySourceHasTokens(searchSources, opts.tenantId ?? null, orgScope)
         : false
       const searchRuntime: SearchRuntime = { ...searchRuntimeBase, searchSources, enabled: searchEnabled && hasSearchTokens }
-      if (
-        searchRuntime.enabled &&
-        searchConfig.useIlikeForNonEncryptedFields === true &&
-        sourceSearchFilters.some((filter) => !String(filter.field).startsWith('cf:'))
-      ) {
-        // `ignoreRuntimeHealth` asks the on-disk question -- a column holds ciphertext even while
-        // the KMS is down -- so an outage keeps encrypted columns on the token path (#4622).
-        // `organizationId: null` is deliberate, not an omission: the service then unions in every
-        // organization's map (`fetchAllOrganizationFieldNames`), so a field any org encrypts stays
-        // on the token path -- a wider set fails safe. Passing the request's org instead would
-        // silently break encrypted-column search for orgs without their own map. That union is an
-        // UNCACHED `encryption_maps` read, one extra round-trip per searched list request.
-        try {
-          const encryptionService = this.getEncryptionService()
-          const readEncryptedFieldNames = encryptionService?.getEncryptedFieldNames?.bind(encryptionService)
-          if (readEncryptedFieldNames) {
-            searchRuntime.encryptedFields = await resolveEncryptedLikeFieldSet(
-              () => readEncryptedFieldNames(
-                entity as EntityId,
-                opts.tenantId ?? null,
-                null,
-                { ignoreRuntimeHealth: true },
-              ),
-              String(entity),
-              opts.tenantId ?? null,
-            )
-          } else if (isTenantDataEncryptionEnabled()) {
-            // Encryption is on but the service is unreachable (a swallowed DI failure looks
-            // exactly like "no service"): treat the map as UNKNOWN and keep the token rewrite,
-            // rather than guessing "plaintext" and running ILIKE against ciphertext.
-            searchRuntime.encryptedFields = null
-          } else {
-            // Encryption disabled: nothing is ciphertext at rest, exact ILIKE is always right.
-            searchRuntime.encryptedFields = new Set()
-          }
-        } catch (err) {
-          // The fallback is safe (the old rewrite-everything behavior), but taking it silently
-          // would hide that the gate has stopped working.
-          logger.warn('search: encrypted-field map unavailable; keeping the token rewrite for all columns', {
-            entity: String(entity),
-            error: err instanceof Error ? err.message : String(err),
-          })
-          searchRuntime.encryptedFields = null
-        }
-      }
       if (searchFilters.length) {
         this.logSearchDebug('search:init', {
           entity,
@@ -845,9 +785,10 @@ export class HybridQueryEngine implements QueryEngine {
         return next
       }
 
-      // Also used by the count shape: cf leaves (and doc-based base leaves) reference
-      // the index rowset aliases, so any group containing one routes the whole
-      // disjunction inside the seeded-rowset EXISTS, where those aliases exist.
+      // Also used by the optimized count path, which builds from a bare base table with
+      // no index/custom-field joins. Emitting a cf predicate there would reference an
+      // alias that query has never joined, so this stays safe only while
+      // `canOptimizeCount` is false whenever any cf filter exists (see `hasCustomFieldFilters`).
       const applyOrGroupedBaseFilters = (q: AnyBuilder): AnyBuilder => {
         if (orGroupFilters.length === 0 && orGroupCfFilters.length === 0) return q
         // `BaseFilter` here is just the normalized-leaf shape; the `cf` bucket holds
@@ -996,101 +937,6 @@ export class HybridQueryEngine implements QueryEngine {
       const hasCustomFieldFilters = cfFilters.length > 0
       const canOptimizeCount = !hasCustomFieldFilters && !hasNonBaseSearchSource
 
-      // ── Count shape (#4552 Phase 2) ─────────────────────────────────
-      // The count query is rebuilt rather than derived from the display shape:
-      // base scope + filters only. Predicates that read the left-joined index
-      // rowset (cf filters, doc filters, or-groups with doc members, token
-      // searches over joined sources) evaluate against the *same* rowset as the
-      // display query, but confined inside a single correlated EXISTS built
-      // from a one-row seed — so the per-base-row rowset is identical, yet a
-      // semi-join cannot multiply base rows. The count therefore needs no
-      // DISTINCT or GROUP BY, and an outer LIMIT actually bounds the scan (no
-      // blocking aggregate between the LIMIT and the scan).
-      const isRowsetDependentFilter = (filter: BaseFilter): boolean => {
-        const baseField = resolveBaseColumn(String(filter.field))
-        if (!baseField) return true
-        // Token search may probe joined sources' record ids, which only exist
-        // on the index rowset.
-        if (
-          (filter.op === 'like' || filter.op === 'ilike') &&
-          searchRuntime.enabled &&
-          typeof filter.value === 'string' &&
-          hasNonBaseSearchSource
-        ) return true
-        return false
-      }
-      const rowsetRegularFilters = regularBaseFilters.filter((filter) => isRowsetDependentFilter(filter))
-      const outerRegularFilters = regularBaseFilters.filter((filter) => !isRowsetDependentFilter(filter))
-      // Or-groups are disjuncts of one $or, so they cannot be split between the
-      // outer query and the rowset — one doc-dependent member moves them all in.
-      const orGroupsRowsetDependent =
-        orGroupCfFilters.length > 0 ||
-        orGroupFilters.some((filter) => isRowsetDependentFilter(filter))
-      const hasInnerTypedCfSource = preparedCfSources.some((source) =>
-        ((opts.customFieldSources ?? []).find((s) => s && (s.alias ?? undefined) === source.alias)?.join?.type ?? 'left') === 'inner')
-      const needsIndexRowset =
-        hasCustomFieldFilters ||
-        rowsetRegularFilters.length > 0 ||
-        orGroupsRowsetDependent ||
-        hasInnerTypedCfSource
-
-      const applyCountShape = async (q: AnyBuilder): Promise<AnyBuilder> => {
-        let next = applyBaseScope(q)
-        for (const filter of outerRegularFilters) {
-          const baseField = resolveBaseColumn(String(filter.field))
-          if (!baseField) continue
-          next = this.applyColumnFilter(next, qualify(baseField), filter, {
-            ...searchRuntime, entity, field: String(filter.field), recordIdColumn: qualify('id'),
-          })
-        }
-        if (!orGroupsRowsetDependent) next = applyOrGroupedBaseFilters(next)
-        if (needsIndexRowset) {
-          // One seed row per base row, then the display query's own join
-          // builders — the rowset inside the EXISTS is the display rowset.
-          // Built from `db` (not the where-callback's expression builder),
-          // matching applyJoinFilters' detached-subquery idiom; correlated
-          // references to the outer base alias resolve at compile time.
-          let rowset: AnyBuilder = db
-            .selectFrom(sql`(select 1)`.as('om_count_seed'))
-            .select(sql<number>`1`.as('one'))
-          rowset = applyEntityIndexesJoin(rowset)
-          rowset = applyCustomFieldSourceJoins(rowset)
-          rowset = applyCfFilters(rowset)
-          for (const filter of rowsetRegularFilters) {
-            const baseField = resolveBaseColumn(String(filter.field))
-            if (!baseField) {
-              rowset = this.applyIndexDocFilterFromAlias(
-                rowset, 'ei', entity, String(filter.field), filter.op, filter.value, qualify('id'), searchRuntime,
-              )
-              continue
-            }
-            rowset = this.applyColumnFilter(rowset, qualify(baseField), filter, {
-              ...searchRuntime, entity, field: String(filter.field), recordIdColumn: qualify('id'),
-            })
-          }
-          if (orGroupsRowsetDependent) rowset = applyOrGroupedBaseFilters(rowset)
-          const capturedRowset = rowset
-          next = next.where((eb: any) => eb.exists(capturedRowset))
-        }
-        next = await applyJoinFilters({
-          db,
-          baseTable,
-          builder: next,
-          joinMap,
-          joinFilters,
-          aliasTables,
-          qualifyBase: (column) => qualify(column),
-          applyAliasScope: async (target: any, alias: string) => applyAliasScopes(target as AnyBuilder, alias),
-          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target as AnyBuilder, column, op, value),
-          applyJoinFilterOp: async (target, filter, qualified, join) => {
-            const applied = await applyJoinSearchFilterOp(target as AnyBuilder, filter, qualified, join)
-            return { applied, builder: target }
-          },
-          columnExists: (tbl, column) => this.columnExists(tbl, column),
-        })
-        return next
-      }
-
       // Selection (for data query)
       const selectFieldSet = new Set<string>((opts.fields && opts.fields.length) ? opts.fields.map(String) : Array.from(columns.keys()))
       if (requiresPlaintextSort) {
@@ -1151,23 +997,33 @@ export class HybridQueryEngine implements QueryEngine {
       const pageSize = opts.page?.pageSize ?? 20
       const sqlDebugEnabled = this.isSqlDebugEnabled()
 
-      const countCap = resolveListCountCap()
+      let total: number
 
-      // Both count paths build the same rebuilt shape; for `canOptimizeCount`
-      // queries `needsIndexRowset` is false, so the shape degenerates to
-      // base-scope + filters exactly as the optimized path always had.
-      // `wasOptimizable` carries NO control flow — it only labels the debug
-      // timing so operators can keep telling the two shapes apart.
-      // TODO(2026-08-12): remove `canOptimizeCount` once the shape convergence
-      // has soaked (tracked in the #4552 spec as the Phase 2 follow-up).
-      const runBoundedCount = async (wasOptimizable: boolean): Promise<{ total: number; warning?: ListCountCapWarning }> => {
-        const countRoot = db.selectFrom(`${baseTable} as b` as any)
-        const shape = await applyCountShape(countRoot)
-        const countQuery = countCap !== null
-          ? db
-              .selectFrom(shape.select(sql<number>`1`.as('one')).limit(countCap + 1).as('om_count_probe') as any)
-              .select(sql<string>`count(*)`.as('count'))
-          : shape.select(sql<string>`count(*)`.as('count'))
+      if (canOptimizeCount) {
+        // Optimized count: apply only base-scope + regular filters + or-group filters (no index joins).
+        const optimizedRoot = db.selectFrom(`${baseTable} as b` as any)
+        let countCore = applyBaseScope(optimizedRoot)
+        countCore = applyRegularBaseFilters(countCore)
+        countCore = applyOrGroupedBaseFilters(countCore)
+        // joinFilters still need to be re-applied in the optimized path
+        countCore = await applyJoinFilters({
+          db,
+          baseTable,
+          builder: countCore,
+          joinMap,
+          joinFilters,
+          aliasTables,
+          qualifyBase: (column) => qualify(column),
+          applyAliasScope: async (target: any, alias: string) => applyAliasScopes(target as AnyBuilder, alias),
+          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target as AnyBuilder, column, op, value),
+          applyJoinFilterOp: async (target, filter, qualified, join) => {
+            const applied = await applyJoinSearchFilterOp(target as AnyBuilder, filter, qualified, join)
+            return { applied, builder: target }
+          },
+          columnExists: (tbl, column) => this.columnExists(tbl, column),
+        })
+        const sub = countCore.select(sql.ref(qualify('id')).as('id')).groupBy(qualify('id')).as('sq')
+        const countQuery = db.selectFrom(sub as any).select(sql<string>`count(*)`.as('count'))
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = countQuery.compile()
           this.debug('query:sql:count', { entity, sql: compiled.sql, bindings: compiled.parameters })
@@ -1175,19 +1031,25 @@ export class HybridQueryEngine implements QueryEngine {
         const countRow = await this.captureSqlTiming(
           'query:sql:count', entity,
           () => countQuery.executeTakeFirst(),
-          { optimized: wasOptimizable }, profiler,
+          { optimized: true }, profiler,
         )
-        const probed = this.parseCount(countRow)
-        if (countCap !== null && probed > countCap) {
-          return { total: countCap, warning: { entity, cap: countCap } }
+        total = this.parseCount(countRow)
+      } else {
+        const countRoot = db.selectFrom(`${baseTable} as b` as any)
+        const countBuilder = (await applyQueryShape(countRoot))
+          .select(sql<string>`count(distinct ${sql.ref(qualify('id'))})`.as('count'))
+        if (debugEnabled && sqlDebugEnabled) {
+          const compiled = countBuilder.compile()
+          this.debug('query:sql:count', { entity, sql: compiled.sql, bindings: compiled.parameters })
         }
-        return { total: probed }
+        const countRow = await this.captureSqlTiming(
+          'query:sql:count', entity,
+          () => countBuilder.executeTakeFirst(),
+          { optimized: false }, profiler,
+        )
+        total = this.parseCount(countRow)
       }
 
-      const counted = await runBoundedCount(canOptimizeCount)
-      const total: number = counted.total
-      const listCountCapWarning: ListCountCapWarning | undefined = counted.warning
-
       const dekKeyCache = new Map<string | null, string | null>()
 
       const decryptRow = async (item: Record<string, unknown>): Promise<Record<string, unknown>> => {
@@ -1238,29 +1100,24 @@ export class HybridQueryEngine implements QueryEngine {
         const sortFieldNames = Array.from(new Set(['id', ...scopeFieldNames, ...resolvedSorts.map((s) => String(s.field))]))
         phase1 = applySelection(phase1, sortFieldNames)
         if (cap !== null) {
-          // Probe one row past the cap: truncation is detected from the candidate
-          // scan itself, not by comparing against `total` — which may itself be
-          // capped (`OM_LIST_COUNT_CAP`) and would then never exceed the sort cap.
-          phase1 = phase1.limit(cap + 1).orderBy(qualify('id'), 'asc' as any)
+          phase1 = phase1.limit(cap).orderBy(qualify('id'), 'asc' as any)
         }
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = phase1.compile()
           this.debug('query:sql:data:phase1', { entity, sql: compiled.sql, bindings: compiled.parameters })
         }
-        const candidateRowsRaw = await this.captureSqlTiming(
+        const candidateRows = await this.captureSqlTiming(
           'query:sql:data:phase1', entity,
           () => phase1.execute(),
           { phase: 1 }, profiler,
         ) as Record<string, unknown>[]
-        const sortTruncated = cap !== null && candidateRowsRaw.length > cap
-        const candidateRows = sortTruncated && cap !== null ? candidateRowsRaw.slice(0, cap) : candidateRowsRaw
         const decryptedCandidates = await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow)
         const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts)
         const pageIds = orderedCandidates
           .slice((page - 1) * pageSize, page * pageSize)
           .map((row) => row.id)
 
-        if (sortTruncated && cap !== null) {
+        if (cap !== null && total > cap) {
           encryptedSortRowCapWarning = {
             entity,
             sortFields: resolvedSorts.map((s) => String(s.field)),
@@ -1316,11 +1173,10 @@ export class HybridQueryEngine implements QueryEngine {
 
       const typedItems = items as unknown as T[]
       let result: QueryResult<T> = { items: typedItems, page, pageSize, total }
-      if (partialIndexWarning || encryptedSortRowCapWarning || listCountCapWarning) {
+      if (partialIndexWarning || encryptedSortRowCapWarning) {
         const meta: QueryResultMeta = {}
         if (partialIndexWarning) meta.partialIndexWarning = partialIndexWarning
         if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning
-        if (listCountCapWarning) meta.listCountCapWarning = listCountCapWarning
         result.meta = meta
       }
 
@@ -1887,13 +1743,11 @@ export class HybridQueryEngine implements QueryEngine {
       return this.buildIndexDocFilterExpression(eb, 'ei', entity, fieldName, filter.op, filter.value, 'b.id', searchRuntime)
     }
     // For like/ilike with active search-tokens, route through hashed-token EXISTS subquery
-    // so encrypted-at-rest columns can still be searched. Plaintext base columns keep exact
-    // SQL ILIKE -- see SearchRuntime.encryptedFields.
+    // so encrypted-at-rest columns can still be searched.
     if (
       (filter.op === 'like' || filter.op === 'ilike') &&
       searchRuntime?.enabled &&
-      typeof filter.value === 'string' &&
-      (searchRuntime.encryptedFields == null || isEncryptedLikeField(searchRuntime.encryptedFields, fieldName))
+      typeof filter.value === 'string'
     ) {
       const tokens = tokenizeText(String(filter.value), searchRuntime.config)
       if (tokens.hashes.length) {
@@ -1917,14 +1771,10 @@ export class HybridQueryEngine implements QueryEngine {
           )
         }
       }
-      // Tokenizer produced no hashes (e.g. value too short) or no source is usable. For a
-      // column KNOWN to be encrypted, `false` is the honest answer for an OR leaf -- `true`
-      // would widen the whole disjunction to match everything, on exactly the columns ILIKE
-      // cannot serve. Every other case (gate off, custom-entity runtime, resolution failure)
-      // keeps the legacy predicate-skipping `true`.
-      return searchRuntime?.encryptedFields != null && isEncryptedLikeField(searchRuntime.encryptedFields, fieldName)
-        ? sql<boolean>`false`
-        : sql<boolean>`true`
+      // Tokenizer produced no hashes (e.g. value too short). Match the regular-base-filter
+      // path's behavior of skipping the predicate (no filter), which is preferable to
+      // silently turning into a plain `ilike` against an encrypted column.
+      return sql<boolean>`true`
     }
     return this.buildColumnFilterExpression(eb, qualify(baseField), filter.op, filter.value)
   }
@@ -2003,9 +1853,6 @@ export class HybridQueryEngine implements QueryEngine {
     const hasSearchTokens = searchEnabled && hasSearchFilter(normalizedFilters)
       ? await this.searchAvailability().hasTokens(entity, opts.tenantId ?? null, orgScope)
       : false
-    // `encryptedFields` is deliberately NOT resolved here: custom-entity rows live in the
-    // `entity_indexes` doc store, whose fields the base-column encryption map does not describe,
-    // so the ILIKE gate stays inert on this path and like/ilike keeps its previous semantics.
     const searchRuntime: SearchRuntime = {
       enabled: searchEnabled && hasSearchTokens,
       config: searchConfig,
@@ -2121,35 +1968,17 @@ export class HybridQueryEngine implements QueryEngine {
     const page = opts.page?.page ?? 1
     const pageSize = opts.page?.pageSize ?? 20
 
-    // Single-table count: `applyScope` adds only WHERE predicates (cf filters
-    // included — no join), and doc storage holds one row per record within a
-    // scope, so `count(*)` needs no DISTINCT and — when the cap is active — a
-    // LIMIT on the row-producing inner query bounds the scan (#4552 Phase 2).
-    const countCap = resolveListCountCap()
     const root = db.selectFrom(`custom_entities_storage as ${alias}`)
-    const countShape = applyScope(root)
-    const countQuery = countCap !== null
-      ? db
-          .selectFrom(countShape.select(sql<number>`1`.as('one')).limit(countCap + 1).as('om_count_probe') as any)
-          .select(sql<string>`count(*)`.as('count'))
-      : countShape.select(sql<string>`count(*)`.as('count'))
+    const countQuery = applyScope(root).select(sql<string>`count(distinct ${sql.ref(`${alias}.entity_id`)})`.as('count'))
     const countRow = await countQuery.executeTakeFirst()
-    const probed = this.parseCount(countRow)
-    let total = probed
-    let listCountCapWarning: ListCountCapWarning | undefined
-    if (countCap !== null && probed > countCap) {
-      total = countCap
-      listCountCapWarning = { entity: entity as EntityId, cap: countCap }
-    }
+    const total = this.parseCount(countRow)
 
     let dataQuery = applyScope(db.selectFrom(`custom_entities_storage as ${alias}`))
     dataQuery = applySelection(dataQuery)
     dataQuery = applySort(dataQuery)
     dataQuery = dataQuery.limit(pageSize).offset((page - 1) * pageSize)
     const items = await dataQuery.execute()
-    const result: QueryResult<T> = { items, page, pageSize, total }
-    if (listCountCapWarning) result.meta = { listCountCapWarning }
-    return result
+    return { items, page, pageSize, total }
   }
 
   private async tableExists(table: string): Promise<boolean> {
@@ -2555,11 +2384,7 @@ export class HybridQueryEngine implements QueryEngine {
     if (
       (filter.op === 'like' || filter.op === 'ilike') &&
       search?.enabled &&
-      typeof filter.value === 'string' &&
-      // Plaintext base columns keep exact SQL ILIKE -- see SearchRuntime.encryptedFields.
-      // Membership runs across name-shape candidates: maps may declare `displayName` while the
-      // filter carries the column name `display_name`.
-      (search.encryptedFields == null || isEncryptedLikeField(search.encryptedFields, search.field))
+      typeof filter.value === 'string'
     ) {
       const tokens = tokenizeText(String(filter.value), search.config)
       const hashes = tokens.hashes
@@ -2588,23 +2413,10 @@ export class HybridQueryEngine implements QueryEngine {
           })
           return q
         }
-        // Hashes exist but no usable search source: same reasoning as the no-hash branch below --
-        // a KNOWN-encrypted column must fail closed rather than drop the predicate (which would
-        // return the full list on exactly the columns ILIKE cannot serve).
-        if (search.encryptedFields != null && isEncryptedLikeField(search.encryptedFields, search.field)) {
-          return q.where(sql<boolean>`false`)
-        }
       } else {
         this.logSearchDebug('search:skip-empty-hashes', {
           entity: search.entity, field: search.field, value: filter.value,
         })
-        // A column KNOWN to be encrypted has no way to match the term except the token index:
-        // dropping the predicate would return every row for a term merely too short to tokenize.
-        // Every other case (gate off, custom-entity runtime, resolution failure) keeps the
-        // legacy behavior of skipping the predicate.
-        if (search.encryptedFields != null && isEncryptedLikeField(search.encryptedFields, search.field)) {
-          return q.where(sql<boolean>`false`)
-        }
       }
       return q
     }

package/src/modules/query_index/lib/search-tokens.ts

@@ -223,49 +223,6 @@ export async function deleteSearchTokensForRecord(
     .execute()
 }
 
-// NUL, not a printable separator: a field name may itself contain a space, so `a b` + hash `c`
-// would otherwise sign identically to field `a` + hash `b c`.
-const SIGNATURE_SEPARATOR = String.fromCharCode(0)
-
-// Identifies one token row for comparison. `token` is NULL unless `storeRawTokens` is on, and a
-// stored NULL has to sign the same as the `null` a freshly built row carries — otherwise every
-// record compares as changed and the skip never fires.
-function tokenSignature(row: { field?: unknown; token_hash?: unknown; token?: unknown }): string {
-  return [
-    String(row.field ?? ''),
-    String(row.token_hash ?? ''),
-    row.token == null ? '' : String(row.token),
-  ].join(SIGNATURE_SEPARATOR)
-}
-
-// Multiplicities, not sets: #4681 reports token rows duplicated by the concurrent-replacement
-// defect, and a set comparison reads such a record as already correct and preserves the duplicates
-// forever. Counting sends it through a full rewrite, which collapses them.
-function tallyEquals(a: Map<string, number> | undefined, b: Map<string, number> | undefined): boolean {
-  const left = a ?? new Map<string, number>()
-  const right = b ?? new Map<string, number>()
-  if (left.size !== right.size) return false
-  for (const [key, count] of left.entries()) {
-    if (right.get(key) !== count) return false
-  }
-  return true
-}
-
-function tallyTokenRows<TRow extends { field?: unknown; token_hash?: unknown; token?: unknown }>(
-  rows: Iterable<TRow>,
-  keyOf: (row: TRow) => string
-): Map<string, Map<string, number>> {
-  const tallies = new Map<string, Map<string, number>>()
-  for (const row of rows) {
-    const key = keyOf(row)
-    const tally = tallies.get(key) ?? new Map<string, number>()
-    const signature = tokenSignature(row)
-    tally.set(signature, (tally.get(signature) ?? 0) + 1)
-    tallies.set(key, tally)
-  }
-  return tallies
-}
-
 export async function replaceSearchTokensForBatch(
   db: Kysely<any>,
   payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>
@@ -299,95 +256,8 @@ export async function replaceSearchTokensForBatch(
     scopeBuckets.set(key, bucket)
   }
 
-  const recordKeyOf = (row: SearchTokenRow) =>
-    `${scopeKey(row.organization_id ?? null, row.tenant_id ?? null)}|${String(row.entity_id)}`
-  const builtTally = tallyTokenRows(rows, recordKeyOf)
-
-  // Read outside the transaction, deliberately. The comparison decides only whether to skip a
-  // rewrite, so a concurrent writer costs us at most a rewrite we declined — declined because the
-  // table already held exactly the rows this call wanted to write. One ordering is worth naming
-  // though: if the read matches and a concurrent writer then commits tokens built from a *staler*
-  // doc, the unconditional rewrite this call used to perform would have overwritten them by
-  // accident. It no longer does, so those stale rows survive until the record's next write. That
-  // is a repair we lose, not a guarantee we break.
-  const changedIdsByBucket = new Map<string, Set<string>>()
-  for (const [key, bucket] of scopeBuckets.entries()) {
-    const ids = Array.from(bucket.ids)
-    const builtCountById = new Map<string, number>()
-    for (const id of ids) {
-      let total = 0
-      const tally = builtTally.get(`${key}|${id}`)
-      if (tally) for (const count of tally.values()) total += count
-      builtCountById.set(id, total)
-    }
-
-    // Count probe first. Its result is one row per record in the batch, so it is bounded by the
-    // batch size — unlike a bare row read, which would be bounded only by how many token rows the
-    // table already holds for these ids, a quantity this function does not control and (per #4681)
-    // has no reason to trust.
-    const storedCounts = await db
-      .selectFrom('search_tokens' as any)
-      .select(['entity_id' as any, sql<number>`count(*)`.as('token_count') as any])
-      .where('entity_type' as any, '=', payloads[0].entityType)
-      .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
-      .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
-      .where('entity_id' as any, 'in', ids)
-      .groupBy('entity_id' as any)
-      .execute()
-    const storedCountById = new Map<string, number>()
-    for (const row of storedCounts as any[]) {
-      storedCountById.set(String(row.entity_id), Number(row.token_count))
-    }
-
-    const changed = new Set<string>()
-    // A record whose stored row count already differs is changed, whatever the rows say — the
-    // duplicate case from #4681 resolves here without ever materializing the duplicated rows.
-    const contentCandidates = ids.filter((id) => {
-      const builtCount = builtCountById.get(id) ?? 0
-      if ((storedCountById.get(id) ?? 0) !== builtCount) {
-        changed.add(id)
-        return false
-      }
-      return builtCount > 0
-    })
-
-    if (contentCandidates.length) {
-      const rowBudget = contentCandidates.reduce((sum, id) => sum + (builtCountById.get(id) ?? 0), 0)
-      const stored = await db
-        .selectFrom('search_tokens' as any)
-        .select(['entity_id' as any, 'field' as any, 'token_hash' as any, 'token' as any])
-        .where('entity_type' as any, '=', payloads[0].entityType)
-        .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
-        .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
-        .where('entity_id' as any, 'in', contentCandidates)
-        // Counts already match, so this cannot truncate — it bounds the damage if a concurrent
-        // writer inserts between the probe and this read. A truncated read compares as changed,
-        // which costs a rewrite rather than a wrong skip.
-        .limit(rowBudget)
-        .execute()
-      const storedTally = tallyTokenRows(stored as any[], (row) => String(row.entity_id))
-      for (const id of contentCandidates) {
-        if (!tallyEquals(builtTally.get(`${key}|${id}`), storedTally.get(id))) changed.add(id)
-      }
-    }
-    changedIdsByBucket.set(key, changed)
-  }
-
-  const changedRecordKeys = new Set<string>()
-  for (const [key, changed] of changedIdsByBucket.entries()) {
-    for (const id of changed) changedRecordKeys.add(`${key}|${id}`)
-  }
-  debug('batch.skip', {
-    entityType: payloads[0].entityType,
-    recordCount: payloads.length,
-    changedCount: changedRecordKeys.size,
-  })
-  if (!changedRecordKeys.size) return
-
   await db.transaction().execute(async (trx) => {
-    for (const [key, bucket] of scopeBuckets.entries()) {
-      const changed = changedIdsByBucket.get(key)
-      if (!changed?.size) continue
+    for (const [, bucket] of scopeBuckets.entries()) {
       // Delete by entity_id: a batch replaces all of a record's tokens, and a per-field OR over the
       // whole batch overflows the query compiler's call stack on large batches.
       const deleteQuery = trx
@@ -395,12 +265,10 @@ export async function replaceSearchTokensForBatch(
         .where('entity_type' as any, '=', payloads[0].entityType)
         .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
         .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
-        .where('entity_id' as any, 'in', Array.from(changed))
+        .where('entity_id' as any, 'in', Array.from(bucket.ids))
       await deleteQuery.execute()
     }
-    const payloadWithTimestamps = rows
-      .filter((row) => changedRecordKeys.has(recordKeyOf(row)))
-      .map((row) => ({ ...row, created_at: sql`now()` }))
+    const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }))
     for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {
       await trx.insertInto('search_tokens' as any).values(batch as any).execute()
     }