@open-mercato/shared 0.6.7-develop.6788.1.c2a1520a30 → 0.6.7-develop.6795.1.8a3f27921c

package/src/lib/search/__tests__/availability.test.ts

@@ -0,0 +1,211 @@
+import {
+  clearSearchTokenPresenceCache,
+  createSearchTokenAvailability,
+  hasSearchFilter,
+  isSearchFilterOp,
+  type OrganizationScope,
+  type SearchTokenProbeQueryBuilder,
+} from '../availability'
+
+beforeEach(() => {
+  clearSearchTokenPresenceCache()
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+afterAll(() => {
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+type ProbeLog = { table: string; wheres: unknown[][] }
+
+function createFakeDb(options?: { rowsByTable?: Record<string, object | undefined>; failTables?: string[] }) {
+  const probes: ProbeLog[] = []
+  const db = {
+    selectFrom(table: string) {
+      const log: ProbeLog = { table, wheres: [] }
+      probes.push(log)
+      const chain = {
+        select: () => chain,
+        where: (...args: unknown[]) => {
+          log.wheres.push(args)
+          return chain
+        },
+        limit: () => chain,
+        executeTakeFirst: async () => {
+          if (options?.failTables?.includes(table)) throw new Error(`probe failed for ${table}`)
+          return options?.rowsByTable?.[table]
+        },
+      }
+      return chain
+    },
+  }
+  return { db, probes }
+}
+
+function buildAvailability(options?: Parameters<typeof createFakeDb>[0] & { enabled?: boolean }) {
+  const { db, probes } = createFakeDb(options)
+  const logDebug = jest.fn()
+  const applyOrganizationScope = jest.fn((query: SearchTokenProbeQueryBuilder) => query)
+  const availability = createSearchTokenAvailability({
+    getDb: () => db,
+    getConfig: () => ({ enabled: options?.enabled ?? true }),
+    applyOrganizationScope,
+    logDebug,
+  })
+  return { availability, probes, logDebug, applyOrganizationScope }
+}
+
+const countProbes = (probes: ProbeLog[], table: string) => probes.filter((probe) => probe.table === table).length
+
+describe('hasSearchFilter / isSearchFilterOp', () => {
+  test('recognizes like and ilike only', () => {
+    expect(isSearchFilterOp('like')).toBe(true)
+    expect(isSearchFilterOp('ilike')).toBe(true)
+    expect(isSearchFilterOp('eq')).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'in' }])).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'ilike' }])).toBe(true)
+    expect(hasSearchFilter([])).toBe(false)
+  })
+})
+
+describe('createSearchTokenAvailability', () => {
+  test('staticEnabled probes the table once per instance and rechecks config each call', async () => {
+    const { availability, probes } = buildAvailability({
+      rowsByTable: { 'information_schema.tables': { one: 1 } },
+    })
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(1)
+  })
+
+  test('staticEnabled short-circuits on disabled config without probing', async () => {
+    const { availability, probes } = buildAvailability({ enabled: false })
+    await expect(availability.staticEnabled()).resolves.toBe(false)
+    expect(probes).toHaveLength(0)
+  })
+
+  test('staticEnabled retries after a failed table probe instead of caching the rejection', async () => {
+    const options = { rowsByTable: { 'information_schema.tables': { one: 1 } }, failTables: ['information_schema.tables'] }
+    const { availability, probes } = buildAvailability(options)
+    await expect(availability.staticEnabled()).rejects.toThrow('probe failed')
+    options.failTables.length = 0
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(2)
+  })
+
+  test('hasTokens memoizes per (entity, tenant, org) key', async () => {
+    const { availability, probes } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: false }
+
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+
+    await availability.hasTokens('example:todo', 't2', orgScope)
+    await availability.hasTokens('example:other', 't1', orgScope)
+    await availability.hasTokens('example:todo', 't1', { ids: ['org2'], includeNull: false })
+    expect(countProbes(probes, 'search_tokens')).toBe(4)
+  })
+
+  test('hasTokens applies the injected organization scope', async () => {
+    const { availability, applyOrganizationScope } = buildAvailability()
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: true }
+    await availability.hasTokens('example:todo', 't1', orgScope)
+    expect(applyOrganizationScope).toHaveBeenCalledWith(expect.anything(), 'search_tokens.organization_id', orgScope)
+
+    applyOrganizationScope.mockClear()
+    await availability.hasTokens('example:todo', 't1', null)
+    expect(applyOrganizationScope).not.toHaveBeenCalled()
+  })
+
+  test('hasTokens resolves false and logs search:has-tokens-error on probe failure', async () => {
+    const { availability, logDebug } = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(logDebug).toHaveBeenCalledWith('search:has-tokens-error', expect.objectContaining({
+      entity: 'example:todo',
+      tenantId: 't1',
+      error: expect.stringContaining('probe failed'),
+    }))
+  })
+
+  test('anySourceHasTokens stops at the first source with tokens and logs each probed source', async () => {
+    const { availability, probes, logDebug } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const result = await availability.anySourceHasTokens(
+      [
+        { entity: 'example:todo', recordIdColumn: 'b.id' },
+        { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+      ],
+      't1',
+      null,
+    )
+    expect(result).toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+    expect(logDebug).toHaveBeenCalledTimes(1)
+    expect(logDebug).toHaveBeenCalledWith('search:source-has-tokens', expect.objectContaining({
+      entity: 'example:todo',
+      recordIdColumn: 'b.id',
+      hasTokens: true,
+    }))
+  })
+
+  test('process-level TTL cache serves later resolver instances without re-probing', async () => {
+    const first = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(first.probes, 'search_tokens')).toBe(1)
+
+    const second = buildAvailability()
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(0)
+  })
+
+  test('process-level TTL cache expires and can be disabled', async () => {
+    const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(1_000_000)
+    try {
+      const first = buildAvailability()
+      await first.availability.hasTokens('example:todo', 't1', null)
+
+      nowSpy.mockReturnValue(1_000_000 + 30_001)
+      const second = buildAvailability()
+      await second.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+
+      process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS = '0'
+      const third = buildAvailability()
+      await third.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(third.probes, 'search_tokens')).toBe(1)
+    } finally {
+      nowSpy.mockRestore()
+    }
+  })
+
+  test('error-driven false results are not TTL-cached', async () => {
+    const first = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+
+    const second = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+  })
+
+  test('probe uses index-friendly tenant predicates instead of IS NOT DISTINCT FROM', async () => {
+    const { availability, probes } = buildAvailability()
+    await availability.hasTokens('example:todo', 't1', null)
+    await availability.hasTokens('example:todo', null, null)
+    const [scoped, global] = probes.filter((probe) => probe.table === 'search_tokens')
+    expect(scoped.wheres).toContainEqual(['tenant_id', '=', 't1'])
+    expect(global.wheres).toContainEqual(['tenant_id', 'is', null])
+  })
+
+  test('anySourceHasTokens sweeps all sources when none has tokens, sharing the memo with hasTokens', async () => {
+    const { availability, probes } = buildAvailability()
+    const sources = [
+      { entity: 'example:todo', recordIdColumn: 'b.id' },
+      { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+    ]
+    await expect(availability.anySourceHasTokens(sources, 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+  })
+})

package/src/lib/search/availability.ts

@@ -0,0 +1,232 @@
+import { sql } from 'kysely'
+import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
+
+export type OrganizationScope = { ids: string[]; includeNull: boolean }
+
+export type SearchTokenSourceRef = { entity: string; recordIdColumn?: string }
+
+type ProbeExpression = object | string | readonly string[] | null
+
+export type SearchTokenProbeQueryBuilder = {
+  select: (selection: ProbeExpression) => SearchTokenProbeQueryBuilder
+  where: (column: ProbeExpression, operator?: string, value?: ProbeExpression) => SearchTokenProbeQueryBuilder
+  limit: (count: number) => SearchTokenProbeQueryBuilder
+  executeTakeFirst: () => Promise<object | undefined>
+}
+
+export type SearchTokenProbeDb = { selectFrom: (table: string) => SearchTokenProbeQueryBuilder }
+
+export type SearchTokenAvailabilityDebugPayload = {
+  entity: string
+  tenantId: string | null
+  organizationScope?: OrganizationScope | null
+  recordIdColumn?: string
+  hasTokens?: boolean
+  error?: string
+}
+
+export type SearchTokenAvailabilityDeps = {
+  getDb: () => SearchTokenProbeDb
+  getConfig: () => { enabled: boolean }
+  applyOrganizationScope: (
+    query: SearchTokenProbeQueryBuilder,
+    column: string,
+    scope: OrganizationScope,
+  ) => SearchTokenProbeQueryBuilder
+  logDebug: (event: string, payload: SearchTokenAvailabilityDebugPayload) => void
+}
+
+export type SearchTokenAvailability = {
+  /**
+   * The cheap, statically-known half of the decision: search is configured on
+   * AND the `search_tokens` table exists. Safe to resolve eagerly (consumers
+   * like custom-field source attachment need it before any filter is
+   * inspected); the table probe is memoized per instance.
+   */
+  staticEnabled: () => Promise<boolean>
+  /**
+   * The expensive half: does `search_tokens` hold any row for this
+   * (entity, tenant, organization scope)? Historically the `LIMIT 1` probe
+   * could degenerate into a seq scan on a large table (#4723), so callers MUST
+   * only ask when the query actually carries a like/ilike filter — use
+   * `hasSearchFilter` for the gate. Three layers keep it cheap for the end
+   * user: index-usable predicates (no `IS NOT DISTINCT FROM`) served by the
+   * dedicated `search_tokens_presence_idx (entity_type, tenant_id,
+   * organization_id)` prefix — which makes the MISS as cheap as the hit — a
+   * per-request instance memo, and a process-level TTL cache
+   * (`OM_SEARCH_TOKEN_PRESENCE_CACHE_MS`, default 30s) that amortizes the
+   * probe across requests. Probe errors log `search:has-tokens-error`,
+   * resolve to `false`, and are never TTL-cached.
+   */
+  hasTokens: (entity: string, tenantId: string | null, orgScope?: OrganizationScope | null) => Promise<boolean>
+  /** First-hit sweep over token sources; logs `search:source-has-tokens` per probed source. */
+  anySourceHasTokens: (
+    sources: SearchTokenSourceRef[],
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ) => Promise<boolean>
+}
+
+export function isSearchFilterOp(op: string | null | undefined): boolean {
+  return op === 'like' || op === 'ilike'
+}
+
+/**
+ * The single definition of "this query actually searches". Every consumer of
+ * the token-availability answer sits behind a like/ilike guard, so when this
+ * returns `false` the `hasTokens` probe's answer would never be read — gate
+ * the probe on it.
+ */
+export function hasSearchFilter(filters: ReadonlyArray<{ op?: string | null }>): boolean {
+  return filters.some((filter) => isSearchFilterOp(filter.op))
+}
+
+function orgScopeKey(scope: OrganizationScope | null | undefined): string {
+  if (!scope) return 'none'
+  return `${scope.includeNull ? '1' : '0'}:${[...scope.ids].sort((left, right) => left.localeCompare(right)).join(',')}`
+}
+
+const PRESENCE_CACHE_DEFAULT_TTL_MS = 30_000
+const PRESENCE_CACHE_MAX_ENTRIES = 10_000
+
+// Process-level TTL cache for the token-presence answer. Module-level (process-global) on
+// purpose: `createRequestContainer` builds fresh engines — and with them fresh resolver
+// instances — per request, so an instance-scoped memo alone re-pays the probe on every
+// request. On a large `search_tokens` one probe can be pathologically expensive (#4723),
+// so the answer is amortized across requests here and only re-checked once per TTL.
+// Staleness contract: a stale `false` keeps like/ilike on the plain-column fallback for up
+// to the TTL after an entity's first tokens are written; a stale `true` routes search
+// through an emptied token set for up to the TTL after a purge. Both converge within the
+// TTL; set OM_SEARCH_TOKEN_PRESENCE_CACHE_MS=0 to disable and probe per request again.
+const presenceCache = new Map<string, { value: boolean; expiresAt: number }>()
+
+function resolvePresenceCacheTtlMs(): number {
+  return parseNumberWithDefault(process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS, PRESENCE_CACHE_DEFAULT_TTL_MS, { integer: true, min: 0 })
+}
+
+function storePresence(key: string, value: boolean, ttlMs: number): void {
+  if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) {
+    const now = Date.now()
+    for (const [entryKey, entry] of presenceCache) {
+      if (entry.expiresAt <= now) presenceCache.delete(entryKey)
+    }
+    if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) presenceCache.clear()
+  }
+  presenceCache.set(key, { value, expiresAt: Date.now() + ttlMs })
+}
+
+export function clearSearchTokenPresenceCache(): void {
+  presenceCache.clear()
+}
+
+/**
+ * One place that answers "is token search usable here?" for both query
+ * engines, instead of each hand-assembling config + table-existence +
+ * token-presence probes with private duplicate helpers and ad-hoc memo maps.
+ *
+ * Memoization is per instance; engines are constructed per request
+ * (`createRequestContainer`), so entries never outlive a request — the same
+ * staleness contract the engines' previous per-query join maps had. A
+ * rejected table probe is evicted so the next call retries instead of
+ * observing a poisoned cache entry.
+ */
+export function createSearchTokenAvailability(deps: SearchTokenAvailabilityDeps): SearchTokenAvailability {
+  const tablePresence = new Map<string, Promise<boolean>>()
+  const tokenPresence = new Map<string, Promise<boolean>>()
+
+  const tableExists = (table: string): Promise<boolean> => {
+    const cached = tablePresence.get(table)
+    if (cached) return cached
+    const probe = (async () => {
+      const row = await deps.getDb()
+        .selectFrom('information_schema.tables')
+        .select(sql<number>`1`.as('one'))
+        .where('table_name', '=', table)
+        .limit(1)
+        .executeTakeFirst()
+      return !!row
+    })()
+    tablePresence.set(table, probe)
+    probe.catch(() => tablePresence.delete(table))
+    return probe
+  }
+
+  const probeTokens = async (
+    entity: string,
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ): Promise<boolean> => {
+    let query = deps.getDb()
+      .selectFrom('search_tokens')
+      .select(sql<number>`1`.as('one'))
+      .where('entity_type', '=', entity)
+    // Deliberately `= / IS NULL` instead of `IS NOT DISTINCT FROM` (identical semantics
+    // for a string|null tenant): the latter cannot serve as an index condition, which is
+    // part of why the planner degraded this probe to a seq scan on large tables (#4723).
+    // With plain predicates the probe is a pure prefix seek on
+    // `search_tokens_presence_idx (entity_type, tenant_id, organization_id)`, making the
+    // miss as cheap as the hit.
+    query = tenantId == null
+      ? query.where('tenant_id', 'is', null)
+      : query.where('tenant_id', '=', tenantId)
+    if (orgScope) {
+      query = deps.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
+    }
+    const row = await query.limit(1).executeTakeFirst()
+    return !!row
+  }
+
+  const hasTokens = (
+    entity: string,
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ): Promise<boolean> => {
+    const key = `${entity}|${tenantId ?? '__null__'}|${orgScopeKey(orgScope)}`
+    const ttlMs = resolvePresenceCacheTtlMs()
+    if (ttlMs > 0) {
+      const entry = presenceCache.get(key)
+      if (entry && entry.expiresAt > Date.now()) return Promise.resolve(entry.value)
+    }
+    const cached = tokenPresence.get(key)
+    if (cached) return cached
+    const probe = (async () => {
+      try {
+        const value = await probeTokens(entity, tenantId, orgScope)
+        // Only genuine probe results enter the process-level cache — caching an
+        // error-driven `false` would pin degraded search for a full TTL after a
+        // transient DB failure.
+        if (ttlMs > 0) storePresence(key, value, ttlMs)
+        return value
+      } catch (err) {
+        deps.logDebug('search:has-tokens-error', {
+          entity,
+          tenantId,
+          organizationScope: orgScope,
+          error: err instanceof Error ? err.message : String(err),
+        })
+        return false
+      }
+    })()
+    tokenPresence.set(key, probe)
+    return probe
+  }
+
+  return {
+    staticEnabled: async () => deps.getConfig().enabled && await tableExists('search_tokens'),
+    hasTokens,
+    anySourceHasTokens: async (sources, tenantId, orgScope) => {
+      for (const source of sources) {
+        const ok = await hasTokens(source.entity, tenantId, orgScope)
+        deps.logDebug('search:source-has-tokens', {
+          entity: source.entity,
+          recordIdColumn: source.recordIdColumn,
+          tenantId,
+          organizationScope: orgScope,
+          hasTokens: ok,
+        })
+        if (ok) return true
+      }
+      return false
+    },
+  }
+}