@open-mercato/core 0.6.7-develop.6863.1.61001cf9a8 → 0.6.7

package/dist/modules/query_index/lib/engine.js

@@ -13,47 +13,13 @@ import {
   resolveJoins
 } from "@open-mercato/shared/lib/query/join-utils";
 import { resolveSearchConfig } from "@open-mercato/shared/lib/search/config";
-import {
-  createSearchTokenAvailability,
-  isSearchFilterOp,
-  hasSearchFilter
-} from "@open-mercato/shared/lib/search/availability";
 import { tokenizeText } from "@open-mercato/shared/lib/search/tokenize";
 import { runBeforeQueryPipeline, runAfterQueryPipeline } from "@open-mercato/shared/lib/query/query-extension-runner";
-import { warnOnCiphertextLikeFallback } from "@open-mercato/shared/lib/query/ciphertext-search-warning";
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "@open-mercato/shared/lib/query/encrypted-sort";
 import { mapWithConcurrency } from "@open-mercato/shared/lib/query/bounded-decrypt";
 import { createLogger } from "@open-mercato/shared/lib/logger";
-import { parseNumberWithDefault } from "@open-mercato/shared/lib/number";
 const logger = createLogger("query_index").child({ component: "engine" });
 const DECRYPT_CONCURRENCY = 8;
-const AUTO_REINDEX_DEBOUNCE_DEFAULT_MS = 3e4;
-const AUTO_REINDEX_DEBOUNCE_MAX_SCOPES = 1e4;
-const AUTO_REINDEX_SCHEDULED_AT_KEY = /* @__PURE__ */ Symbol.for("@open-mercato/query-index/auto-reindex-scheduled-at");
-function getAutoReindexScheduledAt() {
-  const globalScope = globalThis;
-  if (!globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY]) {
-    globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY] = /* @__PURE__ */ new Map();
-  }
-  return globalScope[AUTO_REINDEX_SCHEDULED_AT_KEY];
-}
-function markAutoReindexScheduled(key, debounceMs, now) {
-  const autoReindexScheduledAt = getAutoReindexScheduledAt();
-  const previous = autoReindexScheduledAt.get(key);
-  if (previous !== void 0 && now - previous < debounceMs) return false;
-  if (autoReindexScheduledAt.size >= AUTO_REINDEX_DEBOUNCE_MAX_SCOPES) {
-    for (const [scopeKey, scheduledAt] of autoReindexScheduledAt) {
-      if (now - scheduledAt >= debounceMs) autoReindexScheduledAt.delete(scopeKey);
-    }
-    if (autoReindexScheduledAt.size >= AUTO_REINDEX_DEBOUNCE_MAX_SCOPES) {
-      const oldestKey = autoReindexScheduledAt.keys().next().value;
-      if (oldestKey !== void 0) autoReindexScheduledAt.delete(oldestKey);
-    }
-  }
-  autoReindexScheduledAt.delete(key);
-  autoReindexScheduledAt.set(key, now);
-  return true;
-}
 function buildFilterableCustomFieldJoins(sources) {
   if (!sources || sources.length === 0) return [];
   return sources.flatMap((source, index) => {
@@ -116,10 +82,8 @@ class HybridQueryEngine {
     this.sqlDebugEnabled = null;
     this.forcePartialIndexEnabled = null;
     this.autoReindexEnabled = null;
-    this.autoReindexDebounceMs = null;
     this.coverageOptimizationEnabled = null;
     this.pendingCoverageRefreshKeys = /* @__PURE__ */ new Set();
-    this.searchAvailabilityInstance = null;
     const coverageTtl = Number.parseInt(process.env.QUERY_INDEX_COVERAGE_CACHE_MS ?? "", 10);
     this.coverageStatsTtlMs = Number.isFinite(coverageTtl) && coverageTtl >= 0 ? coverageTtl : 5 * 60 * 1e3;
     const cfTtl = Number.parseInt(process.env.QUERY_INDEX_CF_KEYS_CACHE_MS ?? "", 10);
@@ -137,21 +101,6 @@ class HybridQueryEngine {
     if (typeof emAny.getKysely === "function") return emAny.getKysely();
     throw new Error("HybridQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)");
   }
-  searchAvailability() {
-    if (!this.searchAvailabilityInstance) {
-      this.searchAvailabilityInstance = createSearchTokenAvailability({
-        getDb: () => this.getDb(),
-        getConfig: resolveSearchConfig,
-        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
-          query,
-          column,
-          scope
-        ),
-        logDebug: (event, payload) => this.logSearchDebug(event, payload)
-      });
-    }
-    return this.searchAvailabilityInstance;
-  }
   async query(entity, opts = {}) {
     const ext = opts.extensions;
     let hybridExtCtx = null;
@@ -223,7 +172,7 @@ class HybridQueryEngine {
       profiler.mark("query:base_table_resolved");
       const searchConfig = resolveSearchConfig();
       const orgScope = this.resolveOrganizationScope(opts);
-      const searchEnabled = await this.searchAvailability().staticEnabled();
+      const searchEnabled = searchConfig.enabled && await this.tableExists("search_tokens");
       const baseExists = await profiler.measure("base_table_exists", () => this.tableExists(baseTable));
       if (!baseExists) {
         if (debugEnabled) this.debug("query:fallback:missing-base", { entity, baseTable });
@@ -367,13 +316,10 @@ class HybridQueryEngine {
         }
       }
       const searchSources = indexSources.map((src) => ({ entity: String(src.entityId), recordIdColumn: src.recordIdColumn })).filter((src) => src.recordIdColumn && src.entity);
-      const searchFilters = normalizedFilters.filter((filter) => isSearchFilterOp(filter.op));
-      const sourceSearchFilters = [
-        ...baseFilters,
-        ...normalizedFilters.filter((filter) => String(filter.field).startsWith("cf:"))
-      ].filter((filter) => isSearchFilterOp(filter.op));
-      const hasSearchTokens = searchEnabled && searchSources.length && sourceSearchFilters.length ? await this.searchAvailability().anySourceHasTokens(searchSources, opts.tenantId ?? null, orgScope) : false;
+      const hasSearchTokens = searchEnabled && searchSources.length ? await this.searchSourcesHaveTokens(searchSources, opts.tenantId ?? null, orgScope) : false;
       const searchRuntime = { ...searchRuntimeBase, searchSources, enabled: searchEnabled && hasSearchTokens };
+      const joinSearchAvailability = /* @__PURE__ */ new Map();
+      const searchFilters = normalizeFilters(opts.filters).filter((filter) => filter.op === "like" || filter.op === "ilike");
       if (searchFilters.length) {
         this.logSearchDebug("search:init", {
           entity,
@@ -400,35 +346,6 @@ class HybridQueryEngine {
           organizationScope: orgScope,
           searchSources
         });
-        const baseSearchFilters = [...baseFilters, ...cfFilters].filter((filter) => filter.op === "like" || filter.op === "ilike");
-        const fallbackFields = baseSearchFilters.filter((filter) => !searchRuntime.enabled || typeof filter.value !== "string" || tokenizeText(filter.value, searchConfig).hashes.length === 0).map((filter) => String(filter.field));
-        if (fallbackFields.length) {
-          await warnOnCiphertextLikeFallback({
-            entity: String(entity),
-            fields: fallbackFields,
-            tenantId: opts.tenantId ?? null,
-            // `searchEnabled` also folds in the missing-table case, which is
-            // "no usable tokens" rather than "the operator switched search off".
-            reason: searchRuntime.enabled ? "no-indexable-tokens" : searchConfig.enabled ? "no-search-tokens" : "search-disabled",
-            service: this.getEncryptionService()
-          });
-        }
-      }
-      for (const [alias, joinedFilters] of joinFilters) {
-        const filters = joinedFilters.filter((entry) => entry.op === "like" || entry.op === "ilike");
-        if (!filters.length) continue;
-        const join = joinMap.get(alias);
-        if (!join?.entityId) continue;
-        const hasJoinedTokens = searchEnabled ? await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope) : false;
-        const fallbackFields = filters.filter((filter) => !hasJoinedTokens || typeof filter.value !== "string" || tokenizeText(filter.value, searchConfig).hashes.length === 0).map((filter) => filter.column);
-        if (!fallbackFields.length) continue;
-        await warnOnCiphertextLikeFallback({
-          entity: String(join.entityId),
-          fields: fallbackFields,
-          tenantId: opts.tenantId ?? null,
-          reason: hasJoinedTokens ? "no-indexable-tokens" : searchConfig.enabled ? "no-search-tokens" : "search-disabled",
-          service: this.getEncryptionService()
-        });
       }
       const hasNonBaseSearchSource = searchSources.some(
         (src) => src.entity !== String(entity) || src.recordIdColumn !== "b.id"
@@ -688,7 +605,11 @@ class HybridQueryEngine {
         if (!searchEnabled || !join.entityId) return false;
         if (!["like", "ilike"].includes(filter.op)) return false;
         if (typeof filter.value !== "string" || filter.value.trim().length === 0) return false;
-        const searchAvailable = await this.searchAvailability().hasTokens(String(join.entityId), opts.tenantId ?? null, orgScope);
+        let searchAvailable = joinSearchAvailability.get(join.entityId);
+        if (searchAvailable === void 0) {
+          searchAvailable = await this.hasSearchTokens(String(join.entityId), opts.tenantId ?? null, orgScope);
+          joinSearchAvailability.set(join.entityId, searchAvailable);
+        }
         if (!searchAvailable) return false;
         const tokens = tokenizeText(String(filter.value), searchConfig);
         if (!tokens.hashes.length) return false;
@@ -1435,10 +1356,9 @@ class HybridQueryEngine {
     const alias = "ce";
     const orgScope = this.resolveOrganizationScope(opts);
     if (!opts.tenantId) throw new Error("QueryEngine: tenantId is required");
-    const normalizedFilters = normalizeFilters(opts.filters);
     const searchConfig = resolveSearchConfig();
-    const searchEnabled = await this.searchAvailability().staticEnabled();
-    const hasSearchTokens = searchEnabled && hasSearchFilter(normalizedFilters) ? await this.searchAvailability().hasTokens(entity, opts.tenantId ?? null, orgScope) : false;
+    const searchEnabled = searchConfig.enabled && await this.tableExists("search_tokens");
+    const hasSearchTokens = searchEnabled ? await this.hasSearchTokens(entity, opts.tenantId ?? null, orgScope) : false;
     const searchRuntime = {
       enabled: searchEnabled && hasSearchTokens,
       config: searchConfig,
@@ -1446,6 +1366,7 @@ class HybridQueryEngine {
       tenantId: opts.tenantId ?? null,
       mintAlias: createSearchAliasMinter()
     };
+    const normalizedFilters = normalizeFilters(opts.filters);
     const applyScope = (q) => {
       let next = q.where(`${alias}.entity_type`, "=", entity).where(`${alias}.tenant_id`, "=", opts.tenantId);
       if (orgScope) {
@@ -1558,6 +1479,42 @@ class HybridQueryEngine {
     const exists = await db.selectFrom("information_schema.tables").select(sql`1`.as("one")).where("table_name", "=", table).executeTakeFirst();
     return !!exists;
   }
+  async hasSearchTokens(entity, tenantId, orgScope) {
+    try {
+      const db = this.getDb();
+      let query = db.selectFrom("search_tokens").select(sql`1`.as("one")).where("entity_type", "=", entity);
+      if (tenantId !== void 0) {
+        query = query.where(sql`tenant_id is not distinct from ${tenantId}`);
+      }
+      if (orgScope) {
+        query = this.applyOrganizationScope(query, "search_tokens.organization_id", orgScope);
+      }
+      const row = await query.limit(1).executeTakeFirst();
+      return !!row;
+    } catch (err) {
+      this.logSearchDebug("search:has-tokens-error", {
+        entity,
+        tenantId,
+        organizationScope: orgScope,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      return false;
+    }
+  }
+  async searchSourcesHaveTokens(sources, tenantId, orgScope) {
+    for (const source of sources) {
+      const ok = await this.hasSearchTokens(source.entity, tenantId, orgScope);
+      this.logSearchDebug("search:source-has-tokens", {
+        entity: source.entity,
+        recordIdColumn: source.recordIdColumn,
+        tenantId,
+        organizationScope: orgScope,
+        hasTokens: ok
+      });
+      if (ok) return true;
+    }
+    return false;
+  }
   async resolveAvailableCustomFieldKeys(entityIds, tenantId) {
     if (!entityIds.length) return [];
     const cacheKey = this.customFieldKeysCacheKey(entityIds, tenantId);
@@ -1671,11 +1628,6 @@ class HybridQueryEngine {
       organizationId: organizationIdOverride ?? opts.organizationId ?? null,
       force: false
     };
-    const debounceMs = this.getAutoReindexDebounceMs();
-    if (debounceMs > 0) {
-      const key = [payload.entityType, payload.tenantId ?? "__tenant__", payload.organizationId ?? "__org__"].join("|");
-      if (!markAutoReindexScheduled(key, debounceMs, Date.now())) return;
-    }
     const context = stats ? { entity, tenantId: payload.tenantId, organizationId: payload.organizationId, baseCount: stats.baseCount, indexedCount: stats.indexedCount } : { entity, tenantId: payload.tenantId, organizationId: payload.organizationId };
     void Promise.resolve().then(async () => {
       try {
@@ -1744,15 +1696,6 @@ class HybridQueryEngine {
     this.autoReindexEnabled = parsed === null ? true : parsed;
     return this.autoReindexEnabled;
   }
-  getAutoReindexDebounceMs() {
-    if (this.autoReindexDebounceMs != null) return this.autoReindexDebounceMs;
-    this.autoReindexDebounceMs = parseNumberWithDefault(
-      process.env.OM_QUERY_INDEX_AUTO_REINDEX_DEBOUNCE_MS,
-      AUTO_REINDEX_DEBOUNCE_DEFAULT_MS,
-      { integer: true, min: 0 }
-    );
-    return this.autoReindexDebounceMs;
-  }
   isCoverageOptimizationEnabled() {
     if (this.coverageOptimizationEnabled != null) return this.coverageOptimizationEnabled;
     const raw = (process.env.OPTIMIZE_INDEX_COVERAGE_STATS ?? "").trim().toLowerCase();