@open-mercato/shared 0.6.7-develop.6788.1.c2a1520a30 → 0.6.7-develop.6814.1.0627c7e9f1

package/src/lib/query/engine.ts

@@ -12,6 +12,13 @@ import {
   type ResolvedJoin,
 } from './join-utils'
 import { resolveSearchConfig } from '../search/config'
+import {
+  createSearchTokenAvailability,
+  isSearchFilterOp,
+  type SearchTokenAvailability,
+  type SearchTokenProbeDb,
+  type SearchTokenProbeQueryBuilder,
+} from '../search/availability'
 import { tokenizeText } from '../search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from './query-extension-runner'
 import {
@@ -207,8 +214,8 @@ function computeCustomFieldScore(cfg: Record<string, unknown>, kind: string, ent
  */
 export class BasicQueryEngine implements QueryEngine {
   private columnCache = new Map<string, boolean>()
-  private tableCache = new Map<string, boolean>()
   private searchAliasSeq = 0
+  private searchAvailabilityInstance: SearchTokenAvailability | null = null
 
   constructor(
     private em: EntityManager,
@@ -231,6 +238,22 @@ export class BasicQueryEngine implements QueryEngine {
     throw new Error('BasicQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)')
   }
 
+  private searchAvailability(): SearchTokenAvailability {
+    if (!this.searchAvailabilityInstance) {
+      this.searchAvailabilityInstance = createSearchTokenAvailability({
+        getDb: () => this.getDb() as unknown as SearchTokenProbeDb,
+        getConfig: resolveSearchConfig,
+        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
+          query as unknown as AnyBuilder,
+          column,
+          scope,
+        ) as unknown as SearchTokenProbeQueryBuilder,
+        logDebug: (event, payload) => this.logSearchDebug(event, payload),
+      })
+    }
+    return this.searchAvailabilityInstance
+  }
+
   async query<T = any>(entity: EntityId, opts: QueryOptions = {}): Promise<QueryResult<T>> {
     // --- UMES query extension: before-query pipeline ---
     const ext = opts.extensions
@@ -294,17 +317,20 @@ export class BasicQueryEngine implements QueryEngine {
     const { baseFilters, joinFilters } = partitionFilters(table, normalizedFilters, joinMap)
     const cfFilters = normalizedFilters.filter((filter) => String(filter.field).startsWith('cf:'))
     const searchConfig = resolveSearchConfig()
+    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => isSearchFilterOp(filter.op))
     // Callers that opt out of automatic tenant/org scoping own the full
     // visibility predicate. Search-token filtering has its own tenant/org
     // guards, so it must be disabled on this direct-query path as documented
     // by QueryOptions.omitAutomaticTenantOrgScope.
-    const searchEnabled = !skipAutoScope && searchConfig.enabled && await this.tableExists('search_tokens')
-    const hasSearchTokens = searchEnabled
-      ? await this.hasSearchTokens(String(entity), opts.tenantId ?? null, orgScope)
+    const searchEnabled = !skipAutoScope && await this.searchAvailability().staticEnabled()
+    // Probe `search_tokens` only when this query actually searches (#4723): every consumer of
+    // `searchActive` sits behind a like/ilike guard, so on a plain list load the answer is never
+    // read — and the probe is a `LIMIT 1` the planner can resolve as a seq scan over a large
+    // `search_tokens`. The join path below already probes lazily for the same reason.
+    const hasSearchTokens = searchEnabled && searchFilters.length
+      ? await this.searchAvailability().hasTokens(String(entity), opts.tenantId ?? null, orgScope)
       : false
     const searchActive = searchEnabled && hasSearchTokens
-    const joinSearchAvailability = new Map<string, boolean>()
-    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => filter.op === 'like' || filter.op === 'ilike')
     if (searchFilters.length) {
       const fields = searchFilters.map((filter) => String(filter.field))
       this.logSearchDebug('search:init', {
@@ -358,9 +384,8 @@ export class BasicQueryEngine implements QueryEngine {
       const join = joinMap.get(alias)
       if (!join?.entityId) continue
       const hasJoinedTokens = searchEnabled
-        ? await this.hasSearchTokens(join.entityId, opts.tenantId ?? null, orgScope)
+        ? await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope)
         : false
-      joinSearchAvailability.set(join.entityId, hasJoinedTokens)
       const fallbackFields = filters
         .filter((filter) => !hasJoinedTokens || typeof filter.value !== 'string' || tokenizeText(filter.value, searchConfig).hashes.length === 0)
         .map((filter) => filter.column)
@@ -437,11 +462,7 @@ export class BasicQueryEngine implements QueryEngine {
       if (!['like', 'ilike'].includes(filter.op)) return { applied: false, builder }
       if (typeof filter.value !== 'string' || filter.value.trim().length === 0) return { applied: false, builder }
 
-      let searchAvailable = joinSearchAvailability.get(join.entityId)
-      if (searchAvailable === undefined) {
-        searchAvailable = await this.hasSearchTokens(join.entityId, opts.tenantId ?? null, orgScope)
-        joinSearchAvailability.set(join.entityId, searchAvailable)
-      }
+      const searchAvailable = await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope)
       if (!searchAvailable) return { applied: false, builder }
 
       const tokens = tokenizeText(String(filter.value), searchConfig)
@@ -512,8 +533,8 @@ export class BasicQueryEngine implements QueryEngine {
     // today's complete selection (base fields + CF projections + extension joins).
     // `projection: 'sortKeys'` selects only `id` + the sort columns — the slim phase-1
     // candidate scan used when `requiresPlaintextSort`. Re-running the WHERE/JOIN logic
-    // twice is cheap: every `columnExists`/`tableExists` check is memoized on
-    // `this.columnCache`/`this.tableCache`, so the second pass hits no extra DB calls.
+    // twice is cheap: every `columnExists` check is memoized on `this.columnCache`,
+    // so the second pass hits no extra DB calls.
     const buildQuery = async (projection: 'full' | 'sortKeys'): Promise<BuiltQuery> => {
       const isSortKeysProjection = projection === 'sortKeys'
       let q: AnyBuilder = db.selectFrom(table as any)
@@ -1177,51 +1198,6 @@ export class BasicQueryEngine implements QueryEngine {
     return present
   }
 
-  private async tableExists(table: string): Promise<boolean> {
-    if (this.tableCache.has(table)) return this.tableCache.get(table) ?? false
-    const db = this.getDb()
-    const exists = await db
-      .selectFrom('information_schema.tables' as any)
-      .select(sql<number>`1`.as('one'))
-      .where('table_name' as any, '=', table)
-      .limit(1)
-      .executeTakeFirst()
-    const present = !!exists
-    this.tableCache.set(table, present)
-    return present
-  }
-
-  private async hasSearchTokens(
-    entity: string,
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    try {
-      const db = this.getDb()
-      let query: AnyBuilder = db
-        .selectFrom('search_tokens' as any)
-        .select(sql<number>`1`.as('one'))
-        .where('entity_type' as any, '=', entity)
-        .limit(1)
-      if (tenantId !== undefined) {
-        query = query.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
-      }
-      if (orgScope) {
-        query = this.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
-      }
-      const row = await query.executeTakeFirst()
-      return !!row
-    } catch (err) {
-      this.logSearchDebug('search:has-tokens-error', {
-        entity,
-        tenantId,
-        organizationScope: orgScope,
-        error: err instanceof Error ? err.message : String(err),
-      })
-      return false
-    }
-  }
-
   private applySearchTokens(
     q: AnyBuilder,
     opts: {

package/src/lib/query/types.ts

@@ -108,11 +108,10 @@ export type QueryOptions = {
    * and MUST fail closed when the authenticated principal lacks a resolvable tenant/org, otherwise
    * queries return cross-tenant rows.
    *
-   * When this flag is set, the hybrid query engine delegates to the basic engine, which means
-   * custom-field (`cf:*`) filters/sorts, `search_tokens` fulltext filtering, and vector-search
-   * branches are BYPASSED. Only use this on entities whose scoping does not match the standard
-   * `organization_id = X AND tenant_id = Y` shape and which do not rely on custom-field/search
-   * features.
+   * When this flag is set, the hybrid query engine delegates to the basic engine. The basic engine
+   * still applies `cf:*` filters/sorts, but `search_tokens` fulltext filtering, the JSONB index read
+   * path, and the vector-search branch are BYPASSED. Only use this on entities whose scoping does
+   * not match the standard `organization_id = X AND tenant_id = Y` shape.
    */
   omitAutomaticTenantOrgScope?: boolean
   // Soft-delete behavior: when false (default), rows with non-null deleted_at

package/src/lib/search/__tests__/availability.test.ts

@@ -0,0 +1,211 @@
+import {
+  clearSearchTokenPresenceCache,
+  createSearchTokenAvailability,
+  hasSearchFilter,
+  isSearchFilterOp,
+  type OrganizationScope,
+  type SearchTokenProbeQueryBuilder,
+} from '../availability'
+
+beforeEach(() => {
+  clearSearchTokenPresenceCache()
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+afterAll(() => {
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+type ProbeLog = { table: string; wheres: unknown[][] }
+
+function createFakeDb(options?: { rowsByTable?: Record<string, object | undefined>; failTables?: string[] }) {
+  const probes: ProbeLog[] = []
+  const db = {
+    selectFrom(table: string) {
+      const log: ProbeLog = { table, wheres: [] }
+      probes.push(log)
+      const chain = {
+        select: () => chain,
+        where: (...args: unknown[]) => {
+          log.wheres.push(args)
+          return chain
+        },
+        limit: () => chain,
+        executeTakeFirst: async () => {
+          if (options?.failTables?.includes(table)) throw new Error(`probe failed for ${table}`)
+          return options?.rowsByTable?.[table]
+        },
+      }
+      return chain
+    },
+  }
+  return { db, probes }
+}
+
+function buildAvailability(options?: Parameters<typeof createFakeDb>[0] & { enabled?: boolean }) {
+  const { db, probes } = createFakeDb(options)
+  const logDebug = jest.fn()
+  const applyOrganizationScope = jest.fn((query: SearchTokenProbeQueryBuilder) => query)
+  const availability = createSearchTokenAvailability({
+    getDb: () => db,
+    getConfig: () => ({ enabled: options?.enabled ?? true }),
+    applyOrganizationScope,
+    logDebug,
+  })
+  return { availability, probes, logDebug, applyOrganizationScope }
+}
+
+const countProbes = (probes: ProbeLog[], table: string) => probes.filter((probe) => probe.table === table).length
+
+describe('hasSearchFilter / isSearchFilterOp', () => {
+  test('recognizes like and ilike only', () => {
+    expect(isSearchFilterOp('like')).toBe(true)
+    expect(isSearchFilterOp('ilike')).toBe(true)
+    expect(isSearchFilterOp('eq')).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'in' }])).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'ilike' }])).toBe(true)
+    expect(hasSearchFilter([])).toBe(false)
+  })
+})
+
+describe('createSearchTokenAvailability', () => {
+  test('staticEnabled probes the table once per instance and rechecks config each call', async () => {
+    const { availability, probes } = buildAvailability({
+      rowsByTable: { 'information_schema.tables': { one: 1 } },
+    })
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(1)
+  })
+
+  test('staticEnabled short-circuits on disabled config without probing', async () => {
+    const { availability, probes } = buildAvailability({ enabled: false })
+    await expect(availability.staticEnabled()).resolves.toBe(false)
+    expect(probes).toHaveLength(0)
+  })
+
+  test('staticEnabled retries after a failed table probe instead of caching the rejection', async () => {
+    const options = { rowsByTable: { 'information_schema.tables': { one: 1 } }, failTables: ['information_schema.tables'] }
+    const { availability, probes } = buildAvailability(options)
+    await expect(availability.staticEnabled()).rejects.toThrow('probe failed')
+    options.failTables.length = 0
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(2)
+  })
+
+  test('hasTokens memoizes per (entity, tenant, org) key', async () => {
+    const { availability, probes } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: false }
+
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+
+    await availability.hasTokens('example:todo', 't2', orgScope)
+    await availability.hasTokens('example:other', 't1', orgScope)
+    await availability.hasTokens('example:todo', 't1', { ids: ['org2'], includeNull: false })
+    expect(countProbes(probes, 'search_tokens')).toBe(4)
+  })
+
+  test('hasTokens applies the injected organization scope', async () => {
+    const { availability, applyOrganizationScope } = buildAvailability()
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: true }
+    await availability.hasTokens('example:todo', 't1', orgScope)
+    expect(applyOrganizationScope).toHaveBeenCalledWith(expect.anything(), 'search_tokens.organization_id', orgScope)
+
+    applyOrganizationScope.mockClear()
+    await availability.hasTokens('example:todo', 't1', null)
+    expect(applyOrganizationScope).not.toHaveBeenCalled()
+  })
+
+  test('hasTokens resolves false and logs search:has-tokens-error on probe failure', async () => {
+    const { availability, logDebug } = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(logDebug).toHaveBeenCalledWith('search:has-tokens-error', expect.objectContaining({
+      entity: 'example:todo',
+      tenantId: 't1',
+      error: expect.stringContaining('probe failed'),
+    }))
+  })
+
+  test('anySourceHasTokens stops at the first source with tokens and logs each probed source', async () => {
+    const { availability, probes, logDebug } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const result = await availability.anySourceHasTokens(
+      [
+        { entity: 'example:todo', recordIdColumn: 'b.id' },
+        { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+      ],
+      't1',
+      null,
+    )
+    expect(result).toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+    expect(logDebug).toHaveBeenCalledTimes(1)
+    expect(logDebug).toHaveBeenCalledWith('search:source-has-tokens', expect.objectContaining({
+      entity: 'example:todo',
+      recordIdColumn: 'b.id',
+      hasTokens: true,
+    }))
+  })
+
+  test('process-level TTL cache serves later resolver instances without re-probing', async () => {
+    const first = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(first.probes, 'search_tokens')).toBe(1)
+
+    const second = buildAvailability()
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(0)
+  })
+
+  test('process-level TTL cache expires and can be disabled', async () => {
+    const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(1_000_000)
+    try {
+      const first = buildAvailability()
+      await first.availability.hasTokens('example:todo', 't1', null)
+
+      nowSpy.mockReturnValue(1_000_000 + 30_001)
+      const second = buildAvailability()
+      await second.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+
+      process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS = '0'
+      const third = buildAvailability()
+      await third.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(third.probes, 'search_tokens')).toBe(1)
+    } finally {
+      nowSpy.mockRestore()
+    }
+  })
+
+  test('error-driven false results are not TTL-cached', async () => {
+    const first = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+
+    const second = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+  })
+
+  test('probe uses index-friendly tenant predicates instead of IS NOT DISTINCT FROM', async () => {
+    const { availability, probes } = buildAvailability()
+    await availability.hasTokens('example:todo', 't1', null)
+    await availability.hasTokens('example:todo', null, null)
+    const [scoped, global] = probes.filter((probe) => probe.table === 'search_tokens')
+    expect(scoped.wheres).toContainEqual(['tenant_id', '=', 't1'])
+    expect(global.wheres).toContainEqual(['tenant_id', 'is', null])
+  })
+
+  test('anySourceHasTokens sweeps all sources when none has tokens, sharing the memo with hasTokens', async () => {
+    const { availability, probes } = buildAvailability()
+    const sources = [
+      { entity: 'example:todo', recordIdColumn: 'b.id' },
+      { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+    ]
+    await expect(availability.anySourceHasTokens(sources, 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+  })
+})

package/src/lib/search/__tests__/config.test.ts

@@ -1,8 +1,12 @@
 import {
+  DEFAULT_SEARCH_MAX_FIELD_CHARS,
+  DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+  DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD,
   DEFAULT_SEARCH_MIN_TOKEN_LENGTH,
   isSearchFieldBlocklisted,
   resolveSearchConfig,
   resolveSearchMinTokenLength,
+  resolveSearchTokenLimits,
 } from '../config'
 
 describe('resolveSearchMinTokenLength', () => {
@@ -109,6 +113,61 @@ describe('OM_SEARCH_FIELD_BLOCKLIST parsing', () => {
   })
 })
 
+describe('search token limits', () => {
+  const variableNames = [
+    'OM_SEARCH_MAX_FIELD_CHARS',
+    'OM_SEARCH_MAX_TOKENS_PER_FIELD',
+    'OM_SEARCH_MAX_TOKENS_PER_RECORD',
+  ] as const
+  const originalValues = Object.fromEntries(variableNames.map((name) => [name, process.env[name]]))
+
+  afterEach(() => {
+    for (const name of variableNames) {
+      const original = originalValues[name]
+      if (original === undefined) delete process.env[name]
+      else process.env[name] = original
+    }
+  })
+
+  it('uses safe defaults when limits are unset', () => {
+    for (const name of variableNames) delete process.env[name]
+
+    expect(resolveSearchTokenLimits(resolveSearchConfig())).toEqual({
+      maxFieldChars: DEFAULT_SEARCH_MAX_FIELD_CHARS,
+      maxTokensPerField: DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+      maxTokensPerRecord: DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD,
+    })
+  })
+
+  it('accepts zero to disable individual limits', () => {
+    for (const name of variableNames) process.env[name] = '0'
+
+    expect(resolveSearchTokenLimits(resolveSearchConfig())).toEqual({
+      maxFieldChars: 0,
+      maxTokensPerField: 0,
+      maxTokensPerRecord: 0,
+    })
+  })
+
+  it('normalizes invalid custom config values to defaults', () => {
+    expect(resolveSearchTokenLimits({
+      enabled: true,
+      minTokenLength: 3,
+      enablePartials: true,
+      hashAlgorithm: 'sha256',
+      storeRawTokens: false,
+      blocklistedFields: [],
+      maxFieldChars: Number.NaN,
+      maxTokensPerField: -1,
+      maxTokensPerRecord: 4.8,
+    })).toEqual({
+      maxFieldChars: DEFAULT_SEARCH_MAX_FIELD_CHARS,
+      maxTokensPerField: DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+      maxTokensPerRecord: 4,
+    })
+  })
+})
+
 describe('isSearchFieldBlocklisted', () => {
   const baseConfig = {
     enabled: true,

package/src/lib/search/__tests__/tokenize.test.ts

@@ -0,0 +1,49 @@
+import type { SearchConfig } from '../config'
+import { tokenizeText } from '../tokenize'
+
+const baseConfig: SearchConfig = {
+  enabled: true,
+  minTokenLength: 3,
+  enablePartials: true,
+  hashAlgorithm: 'sha256',
+  storeRawTokens: false,
+  blocklistedFields: [],
+}
+
+describe('tokenizeText limits', () => {
+  test('truncates oversized field text before tokenizing', () => {
+    const config = { ...baseConfig, enablePartials: false, maxFieldChars: 10 }
+
+    const { tokens } = tokenizeText('aaaa bbbb cccc', config)
+
+    expect(tokens).toEqual(['aaaa', 'bbbb'])
+  })
+
+  test('bounds prefix expansion while collecting tokens', () => {
+    const config = { ...baseConfig, maxFieldChars: 100, maxTokensPerField: 5 }
+
+    const { tokens, hashes } = tokenizeText('a'.repeat(100), config)
+
+    expect(tokens).toEqual(['aaa', 'aaaa', 'aaaaa', 'aaaaaa', 'aaaaaaa'])
+    expect(hashes).toHaveLength(5)
+  })
+
+  test('applies safe defaults to legacy configs without limit fields', () => {
+    const { tokens } = tokenizeText('a'.repeat(50_000), baseConfig)
+
+    expect(tokens).toHaveLength(5_000)
+  })
+
+  test('allows a limit to be disabled explicitly', () => {
+    const config = {
+      ...baseConfig,
+      enablePartials: false,
+      maxFieldChars: 0,
+      maxTokensPerField: 0,
+    }
+
+    const { tokens } = tokenizeText('alpha beta gamma delta', config)
+
+    expect(tokens).toEqual(['alpha', 'beta', 'gamma', 'delta'])
+  })
+})