@open-mercato/core 0.6.7-develop.6785.1.1dd7cfac55 → 0.6.7-develop.6795.1.8a3f27921c

package/src/modules/query_index/lib/engine.ts

@@ -20,6 +20,14 @@ import {
   type ResolvedJoin,
 } from '@open-mercato/shared/lib/query/join-utils'
 import { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'
+import {
+  createSearchTokenAvailability,
+  isSearchFilterOp,
+  hasSearchFilter,
+  type SearchTokenAvailability,
+  type SearchTokenProbeDb,
+  type SearchTokenProbeQueryBuilder,
+} from '@open-mercato/shared/lib/search/availability'
 import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from '@open-mercato/shared/lib/query/query-extension-runner'
 import { warnOnCiphertextLikeFallback } from '@open-mercato/shared/lib/query/ciphertext-search-warning'
@@ -146,6 +154,7 @@ export class HybridQueryEngine implements QueryEngine {
   private autoReindexEnabled: boolean | null = null
   private coverageOptimizationEnabled: boolean | null = null
   private pendingCoverageRefreshKeys = new Set<string>()
+  private searchAvailabilityInstance: SearchTokenAvailability | null = null
 
   constructor(
     private em: EntityManager,
@@ -174,6 +183,22 @@ export class HybridQueryEngine implements QueryEngine {
     throw new Error('HybridQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)')
   }
 
+  private searchAvailability(): SearchTokenAvailability {
+    if (!this.searchAvailabilityInstance) {
+      this.searchAvailabilityInstance = createSearchTokenAvailability({
+        getDb: () => this.getDb() as unknown as SearchTokenProbeDb,
+        getConfig: resolveSearchConfig,
+        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
+          query as unknown as AnyBuilder,
+          column,
+          scope,
+        ) as unknown as SearchTokenProbeQueryBuilder,
+        logDebug: (event, payload) => this.logSearchDebug(event, payload),
+      })
+    }
+    return this.searchAvailabilityInstance
+  }
+
   async query<T = unknown>(entity: EntityId, opts: QueryOptions = {}): Promise<QueryResult<T>> {
     const ext: QueryExtensionsConfig | undefined = opts.extensions
     let hybridExtCtx: QueryExtensionContext | null = null
@@ -251,7 +276,7 @@ export class HybridQueryEngine implements QueryEngine {
       profiler.mark('query:base_table_resolved')
       const searchConfig = resolveSearchConfig()
       const orgScope = this.resolveOrganizationScope(opts)
-      const searchEnabled = searchConfig.enabled && await this.tableExists('search_tokens')
+      const searchEnabled = await this.searchAvailability().staticEnabled()
 
       const baseExists = await profiler.measure('base_table_exists', () => this.tableExists(baseTable))
       if (!baseExists) {
@@ -416,16 +441,19 @@ export class HybridQueryEngine implements QueryEngine {
       const searchSources: SearchTokenSource[] = indexSources
         .map((src) => ({ entity: String(src.entityId), recordIdColumn: src.recordIdColumn }))
         .filter((src) => src.recordIdColumn && src.entity)
-      const searchFilters = normalizedFilters.filter((filter) => filter.op === 'like' || filter.op === 'ilike')
-      // Probe `search_tokens` only when this query actually searches. Every consumer of
+      const searchFilters = normalizedFilters.filter((filter) => isSearchFilterOp(filter.op))
+      const sourceSearchFilters = [
+        ...baseFilters,
+        ...normalizedFilters.filter((filter) => String(filter.field).startsWith('cf:')),
+      ].filter((filter) => isSearchFilterOp(filter.op))
+      // Probe `search_tokens` only when this query actually searches (#4723). Every consumer of
       // `searchRuntime.enabled` already sits behind a like/ilike guard, so on a plain list load the
       // answer is unused — and the probe is a `LIMIT 1` the planner can resolve as a seq scan over a
       // large `search_tokens`. The join path below already probes lazily for the same reason.
-      const hasSearchTokens = searchEnabled && searchSources.length && searchFilters.length
-        ? await this.searchSourcesHaveTokens(searchSources, opts.tenantId ?? null, orgScope)
+      const hasSearchTokens = searchEnabled && searchSources.length && sourceSearchFilters.length
+        ? await this.searchAvailability().anySourceHasTokens(searchSources, opts.tenantId ?? null, orgScope)
         : false
       const searchRuntime: SearchRuntime = { ...searchRuntimeBase, searchSources, enabled: searchEnabled && hasSearchTokens }
-      const joinSearchAvailability = new Map<string, boolean>()
       if (searchFilters.length) {
         this.logSearchDebug('search:init', {
           entity,
@@ -476,9 +504,8 @@ export class HybridQueryEngine implements QueryEngine {
         const join = joinMap.get(alias)
         if (!join?.entityId) continue
         const hasJoinedTokens = searchEnabled
-          ? await this.hasSearchTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
+          ? await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
           : false
-        joinSearchAvailability.set(String(join.entityId), hasJoinedTokens)
         const fallbackFields = filters
           .filter((filter) => !hasJoinedTokens || typeof filter.value !== 'string' || tokenizeText(filter.value, searchConfig).hashes.length === 0)
           .map((filter) => filter.column)
@@ -784,11 +811,7 @@ export class HybridQueryEngine implements QueryEngine {
         if (!['like', 'ilike'].includes(filter.op)) return false
         if (typeof filter.value !== 'string' || filter.value.trim().length === 0) return false
 
-        let searchAvailable = joinSearchAvailability.get(join.entityId)
-        if (searchAvailable === undefined) {
-          searchAvailable = await this.hasSearchTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
-          joinSearchAvailability.set(join.entityId, searchAvailable)
-        }
+        const searchAvailable = await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
         if (!searchAvailable) return false
 
         const tokens = tokenizeText(String(filter.value), searchConfig)
@@ -1683,13 +1706,11 @@ export class HybridQueryEngine implements QueryEngine {
     if (!opts.tenantId) throw new Error('QueryEngine: tenantId is required')
 
     const normalizedFilters = normalizeFilters(opts.filters)
-    const hasSearchFilter = normalizedFilters.some((filter) => filter.op === 'like' || filter.op === 'ilike')
-
     const searchConfig = resolveSearchConfig()
-    const searchEnabled = searchConfig.enabled && await this.tableExists('search_tokens')
-    // Same gate as `query()`: without a like/ilike filter the probe's answer is never read.
-    const hasSearchTokens = searchEnabled && hasSearchFilter
-      ? await this.hasSearchTokens(entity, opts.tenantId ?? null, orgScope)
+    const searchEnabled = await this.searchAvailability().staticEnabled()
+    // Same gate as `query()` (#4723): without a like/ilike filter the probe's answer is never read.
+    const hasSearchTokens = searchEnabled && hasSearchFilter(normalizedFilters)
+      ? await this.searchAvailability().hasTokens(entity, opts.tenantId ?? null, orgScope)
       : false
     const searchRuntime: SearchRuntime = {
       enabled: searchEnabled && hasSearchTokens,
@@ -1829,50 +1850,6 @@ export class HybridQueryEngine implements QueryEngine {
     return !!exists
   }
 
-  private async hasSearchTokens(
-    entity: string,
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    try {
-      const db = this.getDb() as any
-      let query = db
-        .selectFrom('search_tokens')
-        .select(sql<number>`1`.as('one'))
-        .where('entity_type', '=', entity)
-      if (tenantId !== undefined) {
-        query = query.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
-      }
-      if (orgScope) {
-        query = this.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
-      }
-      const row = await query.limit(1).executeTakeFirst()
-      return !!row
-    } catch (err) {
-      this.logSearchDebug('search:has-tokens-error', {
-        entity, tenantId, organizationScope: orgScope,
-        error: err instanceof Error ? err.message : String(err),
-      })
-      return false
-    }
-  }
-
-  private async searchSourcesHaveTokens(
-    sources: SearchTokenSource[],
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    for (const source of sources) {
-      const ok = await this.hasSearchTokens(source.entity, tenantId, orgScope)
-      this.logSearchDebug('search:source-has-tokens', {
-        entity: source.entity, recordIdColumn: source.recordIdColumn,
-        tenantId, organizationScope: orgScope, hasTokens: ok,
-      })
-      if (ok) return true
-    }
-    return false
-  }
-
   private async resolveAvailableCustomFieldKeys(entityIds: string[], tenantId: string | null): Promise<string[]> {
     if (!entityIds.length) return []
     const cacheKey = this.customFieldKeysCacheKey(entityIds, tenantId)

package/src/modules/query_index/migrations/.snapshot-open-mercato.json

@@ -1365,6 +1365,18 @@
           "primary": false,
           "unique": false
         },
+        {
+          "columnNames": [
+            "entity_type",
+            "tenant_id",
+            "organization_id"
+          ],
+          "composite": true,
+          "constraint": false,
+          "keyName": "search_tokens_presence_idx",
+          "primary": false,
+          "unique": false
+        },
         {
           "columnNames": [
             "id"

package/src/modules/query_index/migrations/Migration20260731105052_query_index.ts

@@ -0,0 +1,35 @@
+import { Migration } from '@mikro-orm/migrations';
+
+// Availability-probe index (.ai/specs/2026-07-31-search-token-probe-index.md).
+// The query engines gate like/ilike routing on "does (entity_type, tenant, org)
+// have any search tokens?". Neither existing index serves that probe as an
+// access path — lookup_idx buries tenant_id behind field/token_hash — so on a
+// large search_tokens the planner degraded the probe to a seq scan and the MISS
+// answer had to exhaust every entry for the entity type (p95 12.5 s in
+// production, #4723). With this (entity_type, tenant_id, organization_id)
+// prefix and the index-usable `= / IS NULL` predicates the availability
+// resolver emits, both the hit AND the miss become a B-tree prefix seek.
+//
+// search_tokens is high-churn, so the index is built CONCURRENTLY to avoid
+// blocking writes during the build. CREATE INDEX CONCURRENTLY cannot run
+// inside a transaction, hence isTransactional() => false; the migration runner
+// applies migrations one-by-one, so this opt-out is safe (same pattern as
+// Migration20260611103000_query_index). Drop first so retrying a failed
+// concurrent build removes PostgreSQL's invalid index stub instead of letting
+// IF NOT EXISTS silently accept it.
+export class Migration20260731105052_query_index extends Migration {
+
+  override isTransactional(): boolean {
+    return false;
+  }
+
+  override up(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "search_tokens_presence_idx";`);
+    this.addSql(`create index concurrently "search_tokens_presence_idx" on "search_tokens" ("entity_type", "tenant_id", "organization_id");`);
+  }
+
+  override down(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "search_tokens_presence_idx";`);
+  }
+
+}