@open-mercato/core 0.6.7-develop.6788.1.c2a1520a30 → 0.6.7-develop.6814.1.0627c7e9f1

package/src/modules/query_index/lib/engine.ts

@@ -20,16 +20,60 @@ import {
   type ResolvedJoin,
 } from '@open-mercato/shared/lib/query/join-utils'
 import { resolveSearchConfig, type SearchConfig } from '@open-mercato/shared/lib/search/config'
+import {
+  createSearchTokenAvailability,
+  isSearchFilterOp,
+  hasSearchFilter,
+  type SearchTokenAvailability,
+  type SearchTokenProbeDb,
+  type SearchTokenProbeQueryBuilder,
+} from '@open-mercato/shared/lib/search/availability'
 import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from '@open-mercato/shared/lib/query/query-extension-runner'
 import { warnOnCiphertextLikeFallback } from '@open-mercato/shared/lib/query/ciphertext-search-warning'
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from '@open-mercato/shared/lib/query/encrypted-sort'
 import { mapWithConcurrency } from '@open-mercato/shared/lib/query/bounded-decrypt'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
 
 const logger = createLogger('query_index').child({ component: 'engine' })
 
 const DECRYPT_CONCURRENCY = 8
+const AUTO_REINDEX_DEBOUNCE_DEFAULT_MS = 30_000
+const AUTO_REINDEX_DEBOUNCE_MAX_SCOPES = 10_000
+const AUTO_REINDEX_SCHEDULED_AT_KEY = Symbol.for('@open-mercato/query-index/auto-reindex-scheduled-at')
+
+type GlobalWithAutoReindexSchedule = typeof globalThis & {
+  [AUTO_REINDEX_SCHEDULED_AT_KEY]?: Map<string, number>
+}
+
+function getAutoReindexScheduledAt(): Map<string, number> {
+  const globalScope = globalThis as GlobalWithAutoReindexSchedule
+  if (!globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY]) {
+    globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY] = new Map<string, number>()
+  }
+  return globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY]
+}
+
+function markAutoReindexScheduled(key: string, debounceMs: number, now: number): boolean {
+  const autoReindexScheduledAt = getAutoReindexScheduledAt()
+  const previous = autoReindexScheduledAt.get(key)
+  if (previous !== undefined && now - previous < debounceMs) return false
+
+  if (autoReindexScheduledAt.size >= AUTO_REINDEX_DEBOUNCE_MAX_SCOPES) {
+    for (const [scopeKey, scheduledAt] of autoReindexScheduledAt) {
+      if (now - scheduledAt >= debounceMs) autoReindexScheduledAt.delete(scopeKey)
+    }
+    if (autoReindexScheduledAt.size >= AUTO_REINDEX_DEBOUNCE_MAX_SCOPES) {
+      const oldestKey = autoReindexScheduledAt.keys().next().value
+      if (oldestKey !== undefined) autoReindexScheduledAt.delete(oldestKey)
+    }
+  }
+
+  autoReindexScheduledAt.delete(key)
+  autoReindexScheduledAt.set(key, now)
+  return true
+}
 
 function buildFilterableCustomFieldJoins(
   sources: QueryCustomFieldSource[] | undefined,
@@ -144,8 +188,10 @@ export class HybridQueryEngine implements QueryEngine {
   private sqlDebugEnabled: boolean | null = null
   private forcePartialIndexEnabled: boolean | null = null
   private autoReindexEnabled: boolean | null = null
+  private autoReindexDebounceMs: number | null = null
   private coverageOptimizationEnabled: boolean | null = null
   private pendingCoverageRefreshKeys = new Set<string>()
+  private searchAvailabilityInstance: SearchTokenAvailability | null = null
 
   constructor(
     private em: EntityManager,
@@ -174,6 +220,22 @@ export class HybridQueryEngine implements QueryEngine {
     throw new Error('HybridQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)')
   }
 
+  private searchAvailability(): SearchTokenAvailability {
+    if (!this.searchAvailabilityInstance) {
+      this.searchAvailabilityInstance = createSearchTokenAvailability({
+        getDb: () => this.getDb() as unknown as SearchTokenProbeDb,
+        getConfig: resolveSearchConfig,
+        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
+          query as unknown as AnyBuilder,
+          column,
+          scope,
+        ) as unknown as SearchTokenProbeQueryBuilder,
+        logDebug: (event, payload) => this.logSearchDebug(event, payload),
+      })
+    }
+    return this.searchAvailabilityInstance
+  }
+
   async query<T = unknown>(entity: EntityId, opts: QueryOptions = {}): Promise<QueryResult<T>> {
     const ext: QueryExtensionsConfig | undefined = opts.extensions
     let hybridExtCtx: QueryExtensionContext | null = null
@@ -251,7 +313,7 @@ export class HybridQueryEngine implements QueryEngine {
       profiler.mark('query:base_table_resolved')
       const searchConfig = resolveSearchConfig()
       const orgScope = this.resolveOrganizationScope(opts)
-      const searchEnabled = searchConfig.enabled && await this.tableExists('search_tokens')
+      const searchEnabled = await this.searchAvailability().staticEnabled()
 
       const baseExists = await profiler.measure('base_table_exists', () => this.tableExists(baseTable))
       if (!baseExists) {
@@ -416,16 +478,19 @@ export class HybridQueryEngine implements QueryEngine {
       const searchSources: SearchTokenSource[] = indexSources
         .map((src) => ({ entity: String(src.entityId), recordIdColumn: src.recordIdColumn }))
         .filter((src) => src.recordIdColumn && src.entity)
-      const searchFilters = normalizedFilters.filter((filter) => filter.op === 'like' || filter.op === 'ilike')
-      // Probe `search_tokens` only when this query actually searches. Every consumer of
+      const searchFilters = normalizedFilters.filter((filter) => isSearchFilterOp(filter.op))
+      const sourceSearchFilters = [
+        ...baseFilters,
+        ...normalizedFilters.filter((filter) => String(filter.field).startsWith('cf:')),
+      ].filter((filter) => isSearchFilterOp(filter.op))
+      // Probe `search_tokens` only when this query actually searches (#4723). Every consumer of
       // `searchRuntime.enabled` already sits behind a like/ilike guard, so on a plain list load the
       // answer is unused — and the probe is a `LIMIT 1` the planner can resolve as a seq scan over a
       // large `search_tokens`. The join path below already probes lazily for the same reason.
-      const hasSearchTokens = searchEnabled && searchSources.length && searchFilters.length
-        ? await this.searchSourcesHaveTokens(searchSources, opts.tenantId ?? null, orgScope)
+      const hasSearchTokens = searchEnabled && searchSources.length && sourceSearchFilters.length
+        ? await this.searchAvailability().anySourceHasTokens(searchSources, opts.tenantId ?? null, orgScope)
         : false
       const searchRuntime: SearchRuntime = { ...searchRuntimeBase, searchSources, enabled: searchEnabled && hasSearchTokens }
-      const joinSearchAvailability = new Map<string, boolean>()
       if (searchFilters.length) {
         this.logSearchDebug('search:init', {
           entity,
@@ -476,9 +541,8 @@ export class HybridQueryEngine implements QueryEngine {
         const join = joinMap.get(alias)
         if (!join?.entityId) continue
         const hasJoinedTokens = searchEnabled
-          ? await this.hasSearchTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
+          ? await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
           : false
-        joinSearchAvailability.set(String(join.entityId), hasJoinedTokens)
         const fallbackFields = filters
           .filter((filter) => !hasJoinedTokens || typeof filter.value !== 'string' || tokenizeText(filter.value, searchConfig).hashes.length === 0)
           .map((filter) => filter.column)
@@ -784,11 +848,7 @@ export class HybridQueryEngine implements QueryEngine {
         if (!['like', 'ilike'].includes(filter.op)) return false
         if (typeof filter.value !== 'string' || filter.value.trim().length === 0) return false
 
-        let searchAvailable = joinSearchAvailability.get(join.entityId)
-        if (searchAvailable === undefined) {
-          searchAvailable = await this.hasSearchTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
-          joinSearchAvailability.set(join.entityId, searchAvailable)
-        }
+        const searchAvailable = await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope)
         if (!searchAvailable) return false
 
         const tokens = tokenizeText(String(filter.value), searchConfig)
@@ -1683,13 +1743,11 @@ export class HybridQueryEngine implements QueryEngine {
     if (!opts.tenantId) throw new Error('QueryEngine: tenantId is required')
 
     const normalizedFilters = normalizeFilters(opts.filters)
-    const hasSearchFilter = normalizedFilters.some((filter) => filter.op === 'like' || filter.op === 'ilike')
-
     const searchConfig = resolveSearchConfig()
-    const searchEnabled = searchConfig.enabled && await this.tableExists('search_tokens')
-    // Same gate as `query()`: without a like/ilike filter the probe's answer is never read.
-    const hasSearchTokens = searchEnabled && hasSearchFilter
-      ? await this.hasSearchTokens(entity, opts.tenantId ?? null, orgScope)
+    const searchEnabled = await this.searchAvailability().staticEnabled()
+    // Same gate as `query()` (#4723): without a like/ilike filter the probe's answer is never read.
+    const hasSearchTokens = searchEnabled && hasSearchFilter(normalizedFilters)
+      ? await this.searchAvailability().hasTokens(entity, opts.tenantId ?? null, orgScope)
       : false
     const searchRuntime: SearchRuntime = {
       enabled: searchEnabled && hasSearchTokens,
@@ -1829,50 +1887,6 @@ export class HybridQueryEngine implements QueryEngine {
     return !!exists
   }
 
-  private async hasSearchTokens(
-    entity: string,
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    try {
-      const db = this.getDb() as any
-      let query = db
-        .selectFrom('search_tokens')
-        .select(sql<number>`1`.as('one'))
-        .where('entity_type', '=', entity)
-      if (tenantId !== undefined) {
-        query = query.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
-      }
-      if (orgScope) {
-        query = this.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
-      }
-      const row = await query.limit(1).executeTakeFirst()
-      return !!row
-    } catch (err) {
-      this.logSearchDebug('search:has-tokens-error', {
-        entity, tenantId, organizationScope: orgScope,
-        error: err instanceof Error ? err.message : String(err),
-      })
-      return false
-    }
-  }
-
-  private async searchSourcesHaveTokens(
-    sources: SearchTokenSource[],
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    for (const source of sources) {
-      const ok = await this.hasSearchTokens(source.entity, tenantId, orgScope)
-      this.logSearchDebug('search:source-has-tokens', {
-        entity: source.entity, recordIdColumn: source.recordIdColumn,
-        tenantId, organizationScope: orgScope, hasTokens: ok,
-      })
-      if (ok) return true
-    }
-    return false
-  }
-
   private async resolveAvailableCustomFieldKeys(entityIds: string[], tenantId: string | null): Promise<string[]> {
     if (!entityIds.length) return []
     const cacheKey = this.customFieldKeysCacheKey(entityIds, tenantId)
@@ -2021,6 +2035,11 @@ export class HybridQueryEngine implements QueryEngine {
       organizationId: organizationIdOverride ?? opts.organizationId ?? null,
       force: false,
     }
+    const debounceMs = this.getAutoReindexDebounceMs()
+    if (debounceMs > 0) {
+      const key = [payload.entityType, payload.tenantId ?? '__tenant__', payload.organizationId ?? '__org__'].join('|')
+      if (!markAutoReindexScheduled(key, debounceMs, Date.now())) return
+    }
     const context = stats
       ? { entity, tenantId: payload.tenantId, organizationId: payload.organizationId, baseCount: stats.baseCount, indexedCount: stats.indexedCount }
       : { entity, tenantId: payload.tenantId, organizationId: payload.organizationId }
@@ -2090,6 +2109,16 @@ export class HybridQueryEngine implements QueryEngine {
     return this.autoReindexEnabled
   }
 
+  private getAutoReindexDebounceMs(): number {
+    if (this.autoReindexDebounceMs != null) return this.autoReindexDebounceMs
+    this.autoReindexDebounceMs = parseNumberWithDefault(
+      process.env.OM_QUERY_INDEX_AUTO_REINDEX_DEBOUNCE_MS,
+      AUTO_REINDEX_DEBOUNCE_DEFAULT_MS,
+      { integer: true, min: 0 },
+    )
+    return this.autoReindexDebounceMs
+  }
+
   private isCoverageOptimizationEnabled(): boolean {
     if (this.coverageOptimizationEnabled != null) return this.coverageOptimizationEnabled
     const raw = (process.env.OPTIMIZE_INDEX_COVERAGE_STATS ?? '').trim().toLowerCase()

package/src/modules/query_index/lib/search-tokens.ts

@@ -2,6 +2,7 @@ import { type Kysely, type Transaction, sql } from 'kysely'
 import {
   isSearchFieldBlocklisted,
   resolveSearchConfig,
+  resolveSearchTokenLimits,
   type SearchConfig,
 } from '@open-mercato/shared/lib/search/config'
 import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
@@ -79,9 +80,7 @@ function shouldIndexField(
   if (lower.endsWith('_at')) return false
   if (['created_at', 'updated_at', 'deleted_at', 'tenant_id', 'organization_id'].includes(lower)) return false
   if (isSearchFieldBlocklisted(field, entityType, config)) return false
-  const values = collectTextValues(value)
-  if (!values.length) return false
-  return values.some((text) => tokenizeText(text, config).tokens.length > 0)
+  return collectTextValues(value).some((text) => text.length > 0)
 }
 
 export function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[] {
@@ -95,19 +94,32 @@ export function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[
     organizationId: params.organizationId ?? DEFAULT_SCOPE.organizationId,
     tenantId: params.tenantId ?? DEFAULT_SCOPE.tenantId,
   }
+  const limits = resolveSearchTokenLimits(config)
+  const recordLimit = limits.maxTokensPerRecord > 0 ? limits.maxTokensPerRecord : Number.POSITIVE_INFINITY
+  const fieldLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY
 
   for (const [field, rawValue] of Object.entries(params.doc)) {
+    if (tokens.length >= recordLimit) break
     if (!shouldIndexField(field, rawValue, config, params.entityType)) continue
     const values = collectTextValues(rawValue)
     const seen = new Set<string>()
+    let fieldTokenCount = 0
     for (const text of values) {
-      const { tokens: textTokens, hashes } = tokenizeText(text, config)
+      if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break
+      const remainingLimit = Math.min(recordLimit - tokens.length, fieldLimit - fieldTokenCount)
+      const candidateLimit = fieldTokenCount + remainingLimit
+      const tokenConfig = Number.isFinite(candidateLimit)
+        ? { ...config, maxTokensPerField: candidateLimit }
+        : config
+      const { tokens: textTokens, hashes } = tokenizeText(text, tokenConfig)
       for (let i = 0; i < textTokens.length; i += 1) {
+        if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break
         const token = textTokens[i]
         const hash = hashes[i]
         const dedupeKey = `${field}|${hash}`
         if (seen.has(dedupeKey)) continue
         seen.add(dedupeKey)
+        fieldTokenCount += 1
         debug('token.generated', { entityType: params.entityType, recordId: params.recordId, field, hash })
         tokens.push({
           entity_type: params.entityType,

package/src/modules/query_index/migrations/.snapshot-open-mercato.json

@@ -1365,6 +1365,18 @@
           "primary": false,
           "unique": false
         },
+        {
+          "columnNames": [
+            "entity_type",
+            "tenant_id",
+            "organization_id"
+          ],
+          "composite": true,
+          "constraint": false,
+          "keyName": "search_tokens_presence_idx",
+          "primary": false,
+          "unique": false
+        },
         {
           "columnNames": [
             "id"

package/src/modules/query_index/migrations/Migration20260731105052_query_index.ts

@@ -0,0 +1,35 @@
+import { Migration } from '@mikro-orm/migrations';
+
+// Availability-probe index (.ai/specs/2026-07-31-search-token-probe-index.md).
+// The query engines gate like/ilike routing on "does (entity_type, tenant, org)
+// have any search tokens?". Neither existing index serves that probe as an
+// access path — lookup_idx buries tenant_id behind field/token_hash — so on a
+// large search_tokens the planner degraded the probe to a seq scan and the MISS
+// answer had to exhaust every entry for the entity type (p95 12.5 s in
+// production, #4723). With this (entity_type, tenant_id, organization_id)
+// prefix and the index-usable `= / IS NULL` predicates the availability
+// resolver emits, both the hit AND the miss become a B-tree prefix seek.
+//
+// search_tokens is high-churn, so the index is built CONCURRENTLY to avoid
+// blocking writes during the build. CREATE INDEX CONCURRENTLY cannot run
+// inside a transaction, hence isTransactional() => false; the migration runner
+// applies migrations one-by-one, so this opt-out is safe (same pattern as
+// Migration20260611103000_query_index). Drop first so retrying a failed
+// concurrent build removes PostgreSQL's invalid index stub instead of letting
+// IF NOT EXISTS silently accept it.
+export class Migration20260731105052_query_index extends Migration {
+
+  override isTransactional(): boolean {
+    return false;
+  }
+
+  override up(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "search_tokens_presence_idx";`);
+    this.addSql(`create index concurrently "search_tokens_presence_idx" on "search_tokens" ("entity_type", "tenant_id", "organization_id");`);
+  }
+
+  override down(): void | Promise<void> {
+    this.addSql(`drop index concurrently if exists "search_tokens_presence_idx";`);
+  }
+
+}

package/src/modules/shipping_carriers/api/webhook/[provider]/route.ts

@@ -11,6 +11,7 @@ import { getShippingAdapter } from '../../../lib/adapter-registry'
 import { getShippingCarrierQueue } from '../../../lib/queue'
 import { shippingCarriersTag } from '../../openapi'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import { readBoundedRequestBody, WebhookBodyTooLargeError } from '@open-mercato/shared/lib/webhooks'
 
 const logger = createLogger('shipping_carriers').child({ component: 'webhook' })
 
@@ -51,7 +52,15 @@ export async function POST(req: Request, { params }: { params: Promise<{ provide
   const rateLimitResponse = await checkProviderWebhookRateLimit(container, req, providerKey)
   if (rateLimitResponse) return rateLimitResponse
 
-  const rawBody = await req.text()
+  let rawBody: string
+  try {
+    rawBody = await readBoundedRequestBody(req)
+  } catch (error) {
+    if (error instanceof WebhookBodyTooLargeError) {
+      return NextResponse.json({ error: 'Webhook payload too large' }, { status: 413 })
+    }
+    throw error
+  }
   const headers: Record<string, string> = {}
   req.headers.forEach((value, key) => {
     headers[key] = value
@@ -156,6 +165,7 @@ export const openApi = {
       responses: [
         { status: 202, description: 'Webhook accepted for async processing' },
         { status: 401, description: 'Signature verification failed' },
+        { status: 413, description: 'Webhook payload too large' },
         { status: 404, description: 'Unknown provider' },
       ],
     },