@open-mercato/shared 0.6.7 → 0.6.8-develop.6874.1.982d6097d8

package/src/lib/query/engine.ts

@@ -12,6 +12,13 @@ import {
   type ResolvedJoin,
 } from './join-utils'
 import { resolveSearchConfig } from '../search/config'
+import {
+  createSearchTokenAvailability,
+  isSearchFilterOp,
+  type SearchTokenAvailability,
+  type SearchTokenProbeDb,
+  type SearchTokenProbeQueryBuilder,
+} from '../search/availability'
 import { tokenizeText } from '../search/tokenize'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from './query-extension-runner'
 import {
@@ -20,6 +27,7 @@ import {
   type CustomFieldDefinitionRow,
   type ResolvedCustomFieldDefinitions,
 } from '../crud/custom-field-definition-index'
+import { warnOnCiphertextLikeFallback } from './ciphertext-search-warning'
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from './encrypted-sort'
 import { mapWithConcurrency } from './bounded-decrypt'
 import { createLogger } from '../logger'
@@ -206,8 +214,8 @@ function computeCustomFieldScore(cfg: Record<string, unknown>, kind: string, ent
  */
 export class BasicQueryEngine implements QueryEngine {
   private columnCache = new Map<string, boolean>()
-  private tableCache = new Map<string, boolean>()
   private searchAliasSeq = 0
+  private searchAvailabilityInstance: SearchTokenAvailability | null = null
 
   constructor(
     private em: EntityManager,
@@ -230,6 +238,22 @@ export class BasicQueryEngine implements QueryEngine {
     throw new Error('BasicQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)')
   }
 
+  private searchAvailability(): SearchTokenAvailability {
+    if (!this.searchAvailabilityInstance) {
+      this.searchAvailabilityInstance = createSearchTokenAvailability({
+        getDb: () => this.getDb() as unknown as SearchTokenProbeDb,
+        getConfig: resolveSearchConfig,
+        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
+          query as unknown as AnyBuilder,
+          column,
+          scope,
+        ) as unknown as SearchTokenProbeQueryBuilder,
+        logDebug: (event, payload) => this.logSearchDebug(event, payload),
+      })
+    }
+    return this.searchAvailabilityInstance
+  }
+
   async query<T = any>(entity: EntityId, opts: QueryOptions = {}): Promise<QueryResult<T>> {
     // --- UMES query extension: before-query pipeline ---
     const ext = opts.extensions
@@ -293,17 +317,20 @@ export class BasicQueryEngine implements QueryEngine {
     const { baseFilters, joinFilters } = partitionFilters(table, normalizedFilters, joinMap)
     const cfFilters = normalizedFilters.filter((filter) => String(filter.field).startsWith('cf:'))
     const searchConfig = resolveSearchConfig()
+    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => isSearchFilterOp(filter.op))
     // Callers that opt out of automatic tenant/org scoping own the full
     // visibility predicate. Search-token filtering has its own tenant/org
     // guards, so it must be disabled on this direct-query path as documented
     // by QueryOptions.omitAutomaticTenantOrgScope.
-    const searchEnabled = !skipAutoScope && searchConfig.enabled && await this.tableExists('search_tokens')
-    const hasSearchTokens = searchEnabled
-      ? await this.hasSearchTokens(String(entity), opts.tenantId ?? null, orgScope)
+    const searchEnabled = !skipAutoScope && await this.searchAvailability().staticEnabled()
+    // Probe `search_tokens` only when this query actually searches (#4723): every consumer of
+    // `searchActive` sits behind a like/ilike guard, so on a plain list load the answer is never
+    // read — and the probe is a `LIMIT 1` the planner can resolve as a seq scan over a large
+    // `search_tokens`. The join path below already probes lazily for the same reason.
+    const hasSearchTokens = searchEnabled && searchFilters.length
+      ? await this.searchAvailability().hasTokens(String(entity), opts.tenantId ?? null, orgScope)
       : false
     const searchActive = searchEnabled && hasSearchTokens
-    const joinSearchAvailability = new Map<string, boolean>()
-    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => filter.op === 'like' || filter.op === 'ilike')
     if (searchFilters.length) {
       const fields = searchFilters.map((filter) => String(filter.field))
       this.logSearchDebug('search:init', {
@@ -333,6 +360,45 @@ export class BasicQueryEngine implements QueryEngine {
           organizationScope: orgScope,
         })
       }
+      const fallbackFields = searchFilters
+        .filter((filter) => !searchActive || typeof filter.value !== 'string' || tokenizeText(filter.value, searchConfig).hashes.length === 0)
+        .map((filter) => String(filter.field))
+      if (fallbackFields.length) {
+        await warnOnCiphertextLikeFallback({
+          entity: String(entity),
+          fields: fallbackFields,
+          tenantId: opts.tenantId ?? null,
+          // `searchEnabled` also folds in the missing-table and
+          // omitAutomaticTenantOrgScope cases, which are "no usable tokens"
+          // rather than "the operator switched search off".
+          reason: searchActive
+            ? 'no-indexable-tokens'
+            : searchConfig.enabled ? 'no-search-tokens' : 'search-disabled',
+          service: this.getEncryptionService(),
+        })
+      }
+    }
+    for (const [alias, joinedFilters] of joinFilters) {
+      const filters = joinedFilters.filter((entry) => entry.op === 'like' || entry.op === 'ilike')
+      if (!filters.length) continue
+      const join = joinMap.get(alias)
+      if (!join?.entityId) continue
+      const hasJoinedTokens = searchEnabled
+        ? await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope)
+        : false
+      const fallbackFields = filters
+        .filter((filter) => !hasJoinedTokens || typeof filter.value !== 'string' || tokenizeText(filter.value, searchConfig).hashes.length === 0)
+        .map((filter) => filter.column)
+      if (!fallbackFields.length) continue
+      await warnOnCiphertextLikeFallback({
+        entity: join.entityId,
+        fields: fallbackFields,
+        tenantId: opts.tenantId ?? null,
+        reason: hasJoinedTokens
+          ? 'no-indexable-tokens'
+          : searchConfig.enabled ? 'no-search-tokens' : 'search-disabled',
+        service: this.getEncryptionService(),
+      })
     }
     const recordIdColumn = qualify('id')
 
@@ -396,11 +462,7 @@ export class BasicQueryEngine implements QueryEngine {
       if (!['like', 'ilike'].includes(filter.op)) return { applied: false, builder }
       if (typeof filter.value !== 'string' || filter.value.trim().length === 0) return { applied: false, builder }
 
-      let searchAvailable = joinSearchAvailability.get(join.entityId)
-      if (searchAvailable === undefined) {
-        searchAvailable = await this.hasSearchTokens(join.entityId, opts.tenantId ?? null, orgScope)
-        joinSearchAvailability.set(join.entityId, searchAvailable)
-      }
+      const searchAvailable = await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope)
       if (!searchAvailable) return { applied: false, builder }
 
       const tokens = tokenizeText(String(filter.value), searchConfig)
@@ -471,8 +533,8 @@ export class BasicQueryEngine implements QueryEngine {
     // today's complete selection (base fields + CF projections + extension joins).
     // `projection: 'sortKeys'` selects only `id` + the sort columns — the slim phase-1
     // candidate scan used when `requiresPlaintextSort`. Re-running the WHERE/JOIN logic
-    // twice is cheap: every `columnExists`/`tableExists` check is memoized on
-    // `this.columnCache`/`this.tableCache`, so the second pass hits no extra DB calls.
+    // twice is cheap: every `columnExists` check is memoized on `this.columnCache`,
+    // so the second pass hits no extra DB calls.
     const buildQuery = async (projection: 'full' | 'sortKeys'): Promise<BuiltQuery> => {
       const isSortKeysProjection = projection === 'sortKeys'
       let q: AnyBuilder = db.selectFrom(table as any)
@@ -1136,51 +1198,6 @@ export class BasicQueryEngine implements QueryEngine {
     return present
   }
 
-  private async tableExists(table: string): Promise<boolean> {
-    if (this.tableCache.has(table)) return this.tableCache.get(table) ?? false
-    const db = this.getDb()
-    const exists = await db
-      .selectFrom('information_schema.tables' as any)
-      .select(sql<number>`1`.as('one'))
-      .where('table_name' as any, '=', table)
-      .limit(1)
-      .executeTakeFirst()
-    const present = !!exists
-    this.tableCache.set(table, present)
-    return present
-  }
-
-  private async hasSearchTokens(
-    entity: string,
-    tenantId: string | null,
-    orgScope?: { ids: string[]; includeNull: boolean } | null
-  ): Promise<boolean> {
-    try {
-      const db = this.getDb()
-      let query: AnyBuilder = db
-        .selectFrom('search_tokens' as any)
-        .select(sql<number>`1`.as('one'))
-        .where('entity_type' as any, '=', entity)
-        .limit(1)
-      if (tenantId !== undefined) {
-        query = query.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
-      }
-      if (orgScope) {
-        query = this.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
-      }
-      const row = await query.executeTakeFirst()
-      return !!row
-    } catch (err) {
-      this.logSearchDebug('search:has-tokens-error', {
-        entity,
-        tenantId,
-        organizationScope: orgScope,
-        error: err instanceof Error ? err.message : String(err),
-      })
-      return false
-    }
-  }
-
   private applySearchTokens(
     q: AnyBuilder,
     opts: {

package/src/lib/query/types.ts

@@ -108,11 +108,10 @@ export type QueryOptions = {
    * and MUST fail closed when the authenticated principal lacks a resolvable tenant/org, otherwise
    * queries return cross-tenant rows.
    *
-   * When this flag is set, the hybrid query engine delegates to the basic engine, which means
-   * custom-field (`cf:*`) filters/sorts, `search_tokens` fulltext filtering, and vector-search
-   * branches are BYPASSED. Only use this on entities whose scoping does not match the standard
-   * `organization_id = X AND tenant_id = Y` shape and which do not rely on custom-field/search
-   * features.
+   * When this flag is set, the hybrid query engine delegates to the basic engine. The basic engine
+   * still applies `cf:*` filters/sorts, but `search_tokens` fulltext filtering, the JSONB index read
+   * path, and the vector-search branch are BYPASSED. Only use this on entities whose scoping does
+   * not match the standard `organization_id = X AND tenant_id = Y` shape.
    */
   omitAutomaticTenantOrgScope?: boolean
   // Soft-delete behavior: when false (default), rows with non-null deleted_at

package/src/lib/redis/__tests__/connection.test.ts

@@ -5,9 +5,11 @@ describe('parseRedisUrl', () => {
     expect(parseRedisUrl('rediss://:secret@example.cache.amazonaws.com:6379/2')).toEqual({
       host: 'example.cache.amazonaws.com',
       port: 6379,
+      username: undefined,
       password: 'secret',
       db: 2,
       tls: {},
+      family: undefined,
     })
   })
 
@@ -15,9 +17,23 @@ describe('parseRedisUrl', () => {
     expect(parseRedisUrl('redis://:secret@localhost:6379/0')).toEqual({
       host: 'localhost',
       port: 6379,
+      username: undefined,
       password: 'secret',
       db: 0,
       tls: undefined,
+      family: undefined,
+    })
+  })
+
+  it('preserves ACL credentials and supported address-family options', () => {
+    expect(parseRedisUrl('rediss://app%2Duser:p%40ss@example.com:6380/4?family=6')).toEqual({
+      host: 'example.com',
+      port: 6380,
+      username: 'app-user',
+      password: 'p@ss',
+      db: 4,
+      tls: {},
+      family: 6,
     })
   })
 })

package/src/lib/redis/connection.ts

@@ -21,9 +21,11 @@ const logger = createLogger('shared').child({ component: 'redis' })
 export type ParsedRedisConnection = {
   host: string
   port: number
+  username?: string
   password?: string
   db?: number
   tls?: Record<string, unknown>
+  family?: number
 }
 
 /**
@@ -59,12 +61,8 @@ export function getRedisUrlOrThrow(prefix?: string): string {
 }
 
 /**
- * Parse a redis:// URL into a {host, port, password, db} object
- * suitable for BullMQ / ioredis structured connection options.
- *
- * @deprecated Prefer passing the full URL via `{ url: getRedisUrl(...) }` to
- * BullMQ/ioredis — this preserves rediss://, username, database, and query
- * params that structured parsing may lose. Kept for backward compatibility.
+ * Parse a redis:// URL into structured connection options suitable for
+ * BullMQ and ioredis.
  */
 export function parseRedisUrl(url: string): ParsedRedisConnection {
   try {
@@ -72,12 +70,17 @@ export function parseRedisUrl(url: string): ParsedRedisConnection {
     const dbStr = parsed.pathname ? parsed.pathname.slice(1) : ''
     const dbParsed = dbStr !== '' ? parseInt(dbStr, 10) : NaN
     const db = Number.isNaN(dbParsed) ? undefined : dbParsed
+    const familyValue = parsed.searchParams.get('family')
+    const familyParsed = familyValue !== null ? parseInt(familyValue, 10) : NaN
+    const family = [0, 4, 6].includes(familyParsed) ? familyParsed : undefined
     return {
       host: parsed.hostname || 'localhost',
       port: parseInt(parsed.port, 10) || 6379,
-      password: parsed.password || undefined,
+      username: parsed.username ? decodeURIComponent(parsed.username) : undefined,
+      password: parsed.password ? decodeURIComponent(parsed.password) : undefined,
       db,
       tls: parsed.protocol === 'rediss:' ? {} : undefined,
+      family,
     }
   } catch {
     const safeUrl = url.replace(/\/\/[^:]*:[^@]*@/, '//<redacted>@')

package/src/lib/search/__tests__/availability.test.ts

@@ -0,0 +1,211 @@
+import {
+  clearSearchTokenPresenceCache,
+  createSearchTokenAvailability,
+  hasSearchFilter,
+  isSearchFilterOp,
+  type OrganizationScope,
+  type SearchTokenProbeQueryBuilder,
+} from '../availability'
+
+beforeEach(() => {
+  clearSearchTokenPresenceCache()
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+afterAll(() => {
+  delete process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS
+})
+
+type ProbeLog = { table: string; wheres: unknown[][] }
+
+function createFakeDb(options?: { rowsByTable?: Record<string, object | undefined>; failTables?: string[] }) {
+  const probes: ProbeLog[] = []
+  const db = {
+    selectFrom(table: string) {
+      const log: ProbeLog = { table, wheres: [] }
+      probes.push(log)
+      const chain = {
+        select: () => chain,
+        where: (...args: unknown[]) => {
+          log.wheres.push(args)
+          return chain
+        },
+        limit: () => chain,
+        executeTakeFirst: async () => {
+          if (options?.failTables?.includes(table)) throw new Error(`probe failed for ${table}`)
+          return options?.rowsByTable?.[table]
+        },
+      }
+      return chain
+    },
+  }
+  return { db, probes }
+}
+
+function buildAvailability(options?: Parameters<typeof createFakeDb>[0] & { enabled?: boolean }) {
+  const { db, probes } = createFakeDb(options)
+  const logDebug = jest.fn()
+  const applyOrganizationScope = jest.fn((query: SearchTokenProbeQueryBuilder) => query)
+  const availability = createSearchTokenAvailability({
+    getDb: () => db,
+    getConfig: () => ({ enabled: options?.enabled ?? true }),
+    applyOrganizationScope,
+    logDebug,
+  })
+  return { availability, probes, logDebug, applyOrganizationScope }
+}
+
+const countProbes = (probes: ProbeLog[], table: string) => probes.filter((probe) => probe.table === table).length
+
+describe('hasSearchFilter / isSearchFilterOp', () => {
+  test('recognizes like and ilike only', () => {
+    expect(isSearchFilterOp('like')).toBe(true)
+    expect(isSearchFilterOp('ilike')).toBe(true)
+    expect(isSearchFilterOp('eq')).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'in' }])).toBe(false)
+    expect(hasSearchFilter([{ op: 'eq' }, { op: 'ilike' }])).toBe(true)
+    expect(hasSearchFilter([])).toBe(false)
+  })
+})
+
+describe('createSearchTokenAvailability', () => {
+  test('staticEnabled probes the table once per instance and rechecks config each call', async () => {
+    const { availability, probes } = buildAvailability({
+      rowsByTable: { 'information_schema.tables': { one: 1 } },
+    })
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(1)
+  })
+
+  test('staticEnabled short-circuits on disabled config without probing', async () => {
+    const { availability, probes } = buildAvailability({ enabled: false })
+    await expect(availability.staticEnabled()).resolves.toBe(false)
+    expect(probes).toHaveLength(0)
+  })
+
+  test('staticEnabled retries after a failed table probe instead of caching the rejection', async () => {
+    const options = { rowsByTable: { 'information_schema.tables': { one: 1 } }, failTables: ['information_schema.tables'] }
+    const { availability, probes } = buildAvailability(options)
+    await expect(availability.staticEnabled()).rejects.toThrow('probe failed')
+    options.failTables.length = 0
+    await expect(availability.staticEnabled()).resolves.toBe(true)
+    expect(countProbes(probes, 'information_schema.tables')).toBe(2)
+  })
+
+  test('hasTokens memoizes per (entity, tenant, org) key', async () => {
+    const { availability, probes } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: false }
+
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    await expect(availability.hasTokens('example:todo', 't1', orgScope)).resolves.toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+
+    await availability.hasTokens('example:todo', 't2', orgScope)
+    await availability.hasTokens('example:other', 't1', orgScope)
+    await availability.hasTokens('example:todo', 't1', { ids: ['org2'], includeNull: false })
+    expect(countProbes(probes, 'search_tokens')).toBe(4)
+  })
+
+  test('hasTokens applies the injected organization scope', async () => {
+    const { availability, applyOrganizationScope } = buildAvailability()
+    const orgScope: OrganizationScope = { ids: ['org1'], includeNull: true }
+    await availability.hasTokens('example:todo', 't1', orgScope)
+    expect(applyOrganizationScope).toHaveBeenCalledWith(expect.anything(), 'search_tokens.organization_id', orgScope)
+
+    applyOrganizationScope.mockClear()
+    await availability.hasTokens('example:todo', 't1', null)
+    expect(applyOrganizationScope).not.toHaveBeenCalled()
+  })
+
+  test('hasTokens resolves false and logs search:has-tokens-error on probe failure', async () => {
+    const { availability, logDebug } = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(logDebug).toHaveBeenCalledWith('search:has-tokens-error', expect.objectContaining({
+      entity: 'example:todo',
+      tenantId: 't1',
+      error: expect.stringContaining('probe failed'),
+    }))
+  })
+
+  test('anySourceHasTokens stops at the first source with tokens and logs each probed source', async () => {
+    const { availability, probes, logDebug } = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    const result = await availability.anySourceHasTokens(
+      [
+        { entity: 'example:todo', recordIdColumn: 'b.id' },
+        { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+      ],
+      't1',
+      null,
+    )
+    expect(result).toBe(true)
+    expect(countProbes(probes, 'search_tokens')).toBe(1)
+    expect(logDebug).toHaveBeenCalledTimes(1)
+    expect(logDebug).toHaveBeenCalledWith('search:source-has-tokens', expect.objectContaining({
+      entity: 'example:todo',
+      recordIdColumn: 'b.id',
+      hasTokens: true,
+    }))
+  })
+
+  test('process-level TTL cache serves later resolver instances without re-probing', async () => {
+    const first = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(first.probes, 'search_tokens')).toBe(1)
+
+    const second = buildAvailability()
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(0)
+  })
+
+  test('process-level TTL cache expires and can be disabled', async () => {
+    const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(1_000_000)
+    try {
+      const first = buildAvailability()
+      await first.availability.hasTokens('example:todo', 't1', null)
+
+      nowSpy.mockReturnValue(1_000_000 + 30_001)
+      const second = buildAvailability()
+      await second.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+
+      process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS = '0'
+      const third = buildAvailability()
+      await third.availability.hasTokens('example:todo', 't1', null)
+      expect(countProbes(third.probes, 'search_tokens')).toBe(1)
+    } finally {
+      nowSpy.mockRestore()
+    }
+  })
+
+  test('error-driven false results are not TTL-cached', async () => {
+    const first = buildAvailability({ failTables: ['search_tokens'] })
+    await expect(first.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+
+    const second = buildAvailability({ rowsByTable: { search_tokens: { one: 1 } } })
+    await expect(second.availability.hasTokens('example:todo', 't1', null)).resolves.toBe(true)
+    expect(countProbes(second.probes, 'search_tokens')).toBe(1)
+  })
+
+  test('probe uses index-friendly tenant predicates instead of IS NOT DISTINCT FROM', async () => {
+    const { availability, probes } = buildAvailability()
+    await availability.hasTokens('example:todo', 't1', null)
+    await availability.hasTokens('example:todo', null, null)
+    const [scoped, global] = probes.filter((probe) => probe.table === 'search_tokens')
+    expect(scoped.wheres).toContainEqual(['tenant_id', '=', 't1'])
+    expect(global.wheres).toContainEqual(['tenant_id', 'is', null])
+  })
+
+  test('anySourceHasTokens sweeps all sources when none has tokens, sharing the memo with hasTokens', async () => {
+    const { availability, probes } = buildAvailability()
+    const sources = [
+      { entity: 'example:todo', recordIdColumn: 'b.id' },
+      { entity: 'example:other', recordIdColumn: 'cfs0.entity_id' },
+    ]
+    await expect(availability.anySourceHasTokens(sources, 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+
+    await expect(availability.hasTokens('example:todo', 't1', null)).resolves.toBe(false)
+    expect(countProbes(probes, 'search_tokens')).toBe(2)
+  })
+})