@open-mercato/shared 0.6.7 → 0.6.8-develop.6874.1.982d6097d8

package/src/lib/search/availability.ts

@@ -0,0 +1,232 @@
+import { sql } from 'kysely'
+import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
+
+export type OrganizationScope = { ids: string[]; includeNull: boolean }
+
+export type SearchTokenSourceRef = { entity: string; recordIdColumn?: string }
+
+type ProbeExpression = object | string | readonly string[] | null
+
+export type SearchTokenProbeQueryBuilder = {
+  select: (selection: ProbeExpression) => SearchTokenProbeQueryBuilder
+  where: (column: ProbeExpression, operator?: string, value?: ProbeExpression) => SearchTokenProbeQueryBuilder
+  limit: (count: number) => SearchTokenProbeQueryBuilder
+  executeTakeFirst: () => Promise<object | undefined>
+}
+
+export type SearchTokenProbeDb = { selectFrom: (table: string) => SearchTokenProbeQueryBuilder }
+
+export type SearchTokenAvailabilityDebugPayload = {
+  entity: string
+  tenantId: string | null
+  organizationScope?: OrganizationScope | null
+  recordIdColumn?: string
+  hasTokens?: boolean
+  error?: string
+}
+
+export type SearchTokenAvailabilityDeps = {
+  getDb: () => SearchTokenProbeDb
+  getConfig: () => { enabled: boolean }
+  applyOrganizationScope: (
+    query: SearchTokenProbeQueryBuilder,
+    column: string,
+    scope: OrganizationScope,
+  ) => SearchTokenProbeQueryBuilder
+  logDebug: (event: string, payload: SearchTokenAvailabilityDebugPayload) => void
+}
+
+export type SearchTokenAvailability = {
+  /**
+   * The cheap, statically-known half of the decision: search is configured on
+   * AND the `search_tokens` table exists. Safe to resolve eagerly (consumers
+   * like custom-field source attachment need it before any filter is
+   * inspected); the table probe is memoized per instance.
+   */
+  staticEnabled: () => Promise<boolean>
+  /**
+   * The expensive half: does `search_tokens` hold any row for this
+   * (entity, tenant, organization scope)? Historically the `LIMIT 1` probe
+   * could degenerate into a seq scan on a large table (#4723), so callers MUST
+   * only ask when the query actually carries a like/ilike filter — use
+   * `hasSearchFilter` for the gate. Three layers keep it cheap for the end
+   * user: index-usable predicates (no `IS NOT DISTINCT FROM`) served by the
+   * dedicated `search_tokens_presence_idx (entity_type, tenant_id,
+   * organization_id)` prefix — which makes the MISS as cheap as the hit — a
+   * per-request instance memo, and a process-level TTL cache
+   * (`OM_SEARCH_TOKEN_PRESENCE_CACHE_MS`, default 30s) that amortizes the
+   * probe across requests. Probe errors log `search:has-tokens-error`,
+   * resolve to `false`, and are never TTL-cached.
+   */
+  hasTokens: (entity: string, tenantId: string | null, orgScope?: OrganizationScope | null) => Promise<boolean>
+  /** First-hit sweep over token sources; logs `search:source-has-tokens` per probed source. */
+  anySourceHasTokens: (
+    sources: SearchTokenSourceRef[],
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ) => Promise<boolean>
+}
+
+export function isSearchFilterOp(op: string | null | undefined): boolean {
+  return op === 'like' || op === 'ilike'
+}
+
+/**
+ * The single definition of "this query actually searches". Every consumer of
+ * the token-availability answer sits behind a like/ilike guard, so when this
+ * returns `false` the `hasTokens` probe's answer would never be read — gate
+ * the probe on it.
+ */
+export function hasSearchFilter(filters: ReadonlyArray<{ op?: string | null }>): boolean {
+  return filters.some((filter) => isSearchFilterOp(filter.op))
+}
+
+function orgScopeKey(scope: OrganizationScope | null | undefined): string {
+  if (!scope) return 'none'
+  return `${scope.includeNull ? '1' : '0'}:${[...scope.ids].sort((left, right) => left.localeCompare(right)).join(',')}`
+}
+
+const PRESENCE_CACHE_DEFAULT_TTL_MS = 30_000
+const PRESENCE_CACHE_MAX_ENTRIES = 10_000
+
+// Process-level TTL cache for the token-presence answer. Module-level (process-global) on
+// purpose: `createRequestContainer` builds fresh engines — and with them fresh resolver
+// instances — per request, so an instance-scoped memo alone re-pays the probe on every
+// request. On a large `search_tokens` one probe can be pathologically expensive (#4723),
+// so the answer is amortized across requests here and only re-checked once per TTL.
+// Staleness contract: a stale `false` keeps like/ilike on the plain-column fallback for up
+// to the TTL after an entity's first tokens are written; a stale `true` routes search
+// through an emptied token set for up to the TTL after a purge. Both converge within the
+// TTL; set OM_SEARCH_TOKEN_PRESENCE_CACHE_MS=0 to disable and probe per request again.
+const presenceCache = new Map<string, { value: boolean; expiresAt: number }>()
+
+function resolvePresenceCacheTtlMs(): number {
+  return parseNumberWithDefault(process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS, PRESENCE_CACHE_DEFAULT_TTL_MS, { integer: true, min: 0 })
+}
+
+function storePresence(key: string, value: boolean, ttlMs: number): void {
+  if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) {
+    const now = Date.now()
+    for (const [entryKey, entry] of presenceCache) {
+      if (entry.expiresAt <= now) presenceCache.delete(entryKey)
+    }
+    if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) presenceCache.clear()
+  }
+  presenceCache.set(key, { value, expiresAt: Date.now() + ttlMs })
+}
+
+export function clearSearchTokenPresenceCache(): void {
+  presenceCache.clear()
+}
+
+/**
+ * One place that answers "is token search usable here?" for both query
+ * engines, instead of each hand-assembling config + table-existence +
+ * token-presence probes with private duplicate helpers and ad-hoc memo maps.
+ *
+ * Memoization is per instance; engines are constructed per request
+ * (`createRequestContainer`), so entries never outlive a request — the same
+ * staleness contract the engines' previous per-query join maps had. A
+ * rejected table probe is evicted so the next call retries instead of
+ * observing a poisoned cache entry.
+ */
+export function createSearchTokenAvailability(deps: SearchTokenAvailabilityDeps): SearchTokenAvailability {
+  const tablePresence = new Map<string, Promise<boolean>>()
+  const tokenPresence = new Map<string, Promise<boolean>>()
+
+  const tableExists = (table: string): Promise<boolean> => {
+    const cached = tablePresence.get(table)
+    if (cached) return cached
+    const probe = (async () => {
+      const row = await deps.getDb()
+        .selectFrom('information_schema.tables')
+        .select(sql<number>`1`.as('one'))
+        .where('table_name', '=', table)
+        .limit(1)
+        .executeTakeFirst()
+      return !!row
+    })()
+    tablePresence.set(table, probe)
+    probe.catch(() => tablePresence.delete(table))
+    return probe
+  }
+
+  const probeTokens = async (
+    entity: string,
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ): Promise<boolean> => {
+    let query = deps.getDb()
+      .selectFrom('search_tokens')
+      .select(sql<number>`1`.as('one'))
+      .where('entity_type', '=', entity)
+    // Deliberately `= / IS NULL` instead of `IS NOT DISTINCT FROM` (identical semantics
+    // for a string|null tenant): the latter cannot serve as an index condition, which is
+    // part of why the planner degraded this probe to a seq scan on large tables (#4723).
+    // With plain predicates the probe is a pure prefix seek on
+    // `search_tokens_presence_idx (entity_type, tenant_id, organization_id)`, making the
+    // miss as cheap as the hit.
+    query = tenantId == null
+      ? query.where('tenant_id', 'is', null)
+      : query.where('tenant_id', '=', tenantId)
+    if (orgScope) {
+      query = deps.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
+    }
+    const row = await query.limit(1).executeTakeFirst()
+    return !!row
+  }
+
+  const hasTokens = (
+    entity: string,
+    tenantId: string | null,
+    orgScope?: OrganizationScope | null,
+  ): Promise<boolean> => {
+    const key = `${entity}|${tenantId ?? '__null__'}|${orgScopeKey(orgScope)}`
+    const ttlMs = resolvePresenceCacheTtlMs()
+    if (ttlMs > 0) {
+      const entry = presenceCache.get(key)
+      if (entry && entry.expiresAt > Date.now()) return Promise.resolve(entry.value)
+    }
+    const cached = tokenPresence.get(key)
+    if (cached) return cached
+    const probe = (async () => {
+      try {
+        const value = await probeTokens(entity, tenantId, orgScope)
+        // Only genuine probe results enter the process-level cache — caching an
+        // error-driven `false` would pin degraded search for a full TTL after a
+        // transient DB failure.
+        if (ttlMs > 0) storePresence(key, value, ttlMs)
+        return value
+      } catch (err) {
+        deps.logDebug('search:has-tokens-error', {
+          entity,
+          tenantId,
+          organizationScope: orgScope,
+          error: err instanceof Error ? err.message : String(err),
+        })
+        return false
+      }
+    })()
+    tokenPresence.set(key, probe)
+    return probe
+  }
+
+  return {
+    staticEnabled: async () => deps.getConfig().enabled && await tableExists('search_tokens'),
+    hasTokens,
+    anySourceHasTokens: async (sources, tenantId, orgScope) => {
+      for (const source of sources) {
+        const ok = await hasTokens(source.entity, tenantId, orgScope)
+        deps.logDebug('search:source-has-tokens', {
+          entity: source.entity,
+          recordIdColumn: source.recordIdColumn,
+          tenantId,
+          organizationScope: orgScope,
+          hasTokens: ok,
+        })
+        if (ok) return true
+      }
+      return false
+    },
+  }
+}

package/src/lib/search/config.ts

@@ -1,5 +1,6 @@
 import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
 import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
+import { parseCommaSeparatedList } from '@open-mercato/shared/lib/string'
 
 export type SearchConfig = {
   enabled: boolean
@@ -8,12 +9,27 @@ export type SearchConfig = {
   hashAlgorithm: 'sha256' | 'sha1' | 'md5'
   storeRawTokens: boolean
   blocklistedFields: string[]
+  entityBlocklistedFields?: Record<string, string[]>
+  maxFieldChars?: number
+  maxTokensPerField?: number
+  maxTokensPerRecord?: number
 }
 
 export const DEFAULT_SEARCH_MIN_TOKEN_LENGTH = 3
+export const DEFAULT_SEARCH_MAX_FIELD_CHARS = 20_000
+export const DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD = 5_000
+export const DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD = 20_000
+
+export type SearchTokenLimits = {
+  maxFieldChars: number
+  maxTokensPerField: number
+  maxTokensPerRecord: number
+}
 
 const DEFAULT_BLOCKLIST = ['password', 'token', 'secret', 'hash']
 
+const ENTITY_BLOCKLIST_SEPARATOR = '@'
+
 function parseBoolean(raw: string | undefined, fallback: boolean): boolean {
   return parseBooleanWithDefault(raw, fallback)
 }
@@ -22,6 +38,19 @@ function parseNumber(raw: string | undefined, fallback: number, min = 1): number
   return parseNumberWithDefault(raw, fallback, { integer: true, min })
 }
 
+export function resolveSearchTokenLimits(config: SearchConfig): SearchTokenLimits {
+  const resolveLimit = (value: number | undefined, fallback: number): number => {
+    if (value === undefined) return fallback
+    if (!Number.isFinite(value) || value < 0) return fallback
+    return Math.trunc(value)
+  }
+  return {
+    maxFieldChars: resolveLimit(config.maxFieldChars, DEFAULT_SEARCH_MAX_FIELD_CHARS),
+    maxTokensPerField: resolveLimit(config.maxTokensPerField, DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD),
+    maxTokensPerRecord: resolveLimit(config.maxTokensPerRecord, DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD),
+  }
+}
+
 function parseHashAlgorithm(raw: string | undefined): 'sha256' | 'sha1' | 'md5' {
   const value = (raw ?? '').trim().toLowerCase()
   if (value === 'sha1') return 'sha1'
@@ -29,24 +58,93 @@ function parseHashAlgorithm(raw: string | undefined): 'sha256' | 'sha1' | 'md5'
   return 'sha256'
 }
 
+/**
+ * Parses `OM_SEARCH_FIELD_BLOCKLIST` into a global list plus per-entity-type lists.
+ *
+ * Why: a deployment often needs to keep one large free-text column out of the token
+ * index (e-mail bodies on `customers:customer_interaction`) while still indexing the
+ * same-named column elsewhere. A flat global list cannot express that.
+ *
+ * How to apply: entries are comma-separated; an entry may carry an optional
+ * `entityType@` prefix — `body` blocks the field everywhere, while
+ * `customers:customer_interaction@body` blocks it only for that entity type. Entries
+ * whose field part is empty are ignored so malformed env input cannot break indexing.
+ */
+function parseFieldBlocklist(raw: string | undefined): {
+  global: string[]
+  byEntity: Record<string, string[]>
+} {
+  const global: string[] = []
+  const byEntity = new Map<string, string[]>()
+
+  for (const rawEntry of parseCommaSeparatedList(raw)) {
+    const entry = rawEntry.toLowerCase()
+    const separatorIndex = entry.indexOf(ENTITY_BLOCKLIST_SEPARATOR)
+    const entityType = separatorIndex >= 0 ? entry.slice(0, separatorIndex).trim() : ''
+    const field = separatorIndex >= 0 ? entry.slice(separatorIndex + 1).trim() : entry
+    if (!field.length) continue
+
+    if (!entityType.length) {
+      if (!global.includes(field)) global.push(field)
+      continue
+    }
+
+    const scoped = byEntity.get(entityType) ?? []
+    if (!scoped.includes(field)) scoped.push(field)
+    byEntity.set(entityType, scoped)
+  }
+
+  for (const fallback of DEFAULT_BLOCKLIST) {
+    if (!global.includes(fallback)) global.push(fallback)
+  }
+
+  const scopedBlocklist = Object.create(null) as Record<string, string[]>
+  for (const [entityType, fields] of byEntity) scopedBlocklist[entityType] = fields
+
+  return { global, byEntity: scopedBlocklist }
+}
+
 export function resolveSearchConfig(): SearchConfig {
+  const blocklist = parseFieldBlocklist(process.env.OM_SEARCH_FIELD_BLOCKLIST)
   return {
     enabled: parseBoolean(process.env.OM_SEARCH_ENABLED, true),
     minTokenLength: resolveSearchMinTokenLength(),
     enablePartials: parseBoolean(process.env.OM_SEARCH_ENABLE_PARTIAL, true),
     hashAlgorithm: parseHashAlgorithm(process.env.OM_SEARCH_HASH_ALGO),
     storeRawTokens: parseBoolean(process.env.OM_SEARCH_STORE_RAW_TOKENS, false),
-    blocklistedFields: (process.env.OM_SEARCH_FIELD_BLOCKLIST ?? '')
-      .split(',')
-      .map((entry) => entry.trim())
-      .filter((entry) => entry.length > 0)
-      .filter((value, index, arr) => arr.indexOf(value) === index)
-      .map((entry) => entry.toLowerCase())
-      .concat(DEFAULT_BLOCKLIST)
-      .filter((value, index, arr) => arr.indexOf(value) === index),
+    blocklistedFields: blocklist.global,
+    entityBlocklistedFields: blocklist.byEntity,
+    maxFieldChars: parseNumber(process.env.OM_SEARCH_MAX_FIELD_CHARS, DEFAULT_SEARCH_MAX_FIELD_CHARS, 0),
+    maxTokensPerField: parseNumber(process.env.OM_SEARCH_MAX_TOKENS_PER_FIELD, DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD, 0),
+    maxTokensPerRecord: parseNumber(process.env.OM_SEARCH_MAX_TOKENS_PER_RECORD, DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD, 0),
   }
 }
 
+/**
+ * Single matcher for "should this field be kept out of the search index?".
+ *
+ * Why: the per-field token path and the `search_text` aggregate previously each
+ * decided this on their own, and the aggregate simply never consulted the config —
+ * so a blocklisted column's text came back into the index under the aggregate's
+ * field name (#4624). Both paths now share this function so they cannot drift.
+ *
+ * How to apply: pass the document's field name and the entity type being indexed;
+ * `entityType` may be omitted when unknown, in which case only global entries apply.
+ * Matching keeps the historical substring semantics (`fieldName.includes(pattern)`).
+ */
+export function isSearchFieldBlocklisted(
+  field: string,
+  entityType: string | null | undefined,
+  config: SearchConfig,
+): boolean {
+  const lower = field.toLowerCase()
+  if (config.blocklistedFields.some((blocked) => lower.includes(blocked))) return true
+  if (!entityType) return false
+  const scoped = config.entityBlocklistedFields?.[entityType.trim().toLowerCase()]
+  if (!Array.isArray(scoped) || !scoped.length) return false
+  return scoped.some((blocked) => lower.includes(blocked))
+}
+
 /**
  * Browser-safe accessor for the minimum search token length.
  *

package/src/lib/search/tokenLookup.ts

@@ -0,0 +1,133 @@
+import { type Kysely, sql } from 'kysely'
+import { resolveSearchConfig, type SearchConfig } from './config'
+import { tokenizeText } from './tokenize'
+
+export type SearchTokenDatabase = {
+  search_tokens: {
+    entity_id: string
+    entity_type: string
+    field: string
+    token_hash: string
+    tenant_id: string | null
+    organization_id: string | null
+  }
+}
+
+/**
+ * Tenant/organization scoping for a `search_tokens` lookup.
+ *
+ * `undefined` and `null` are NOT interchangeable:
+ * - `undefined` omits the predicate entirely (the caller owns visibility).
+ * - `null` emits a null-safe predicate that matches only globally scoped rows.
+ */
+export type SearchTokenScope = {
+  tenantId?: string | null
+  organizationId?: string | null
+  organizationIds?: readonly string[] | null
+}
+
+/**
+ * Why a lookup could not produce an id set. Callers MUST NOT read these as
+ * "nothing matched" — the token index was never consulted, so the caller's own
+ * predicate (usually an `ilike`) is still the authoritative one.
+ */
+export type SearchTokenLookupSkipReason = 'empty-query' | 'search-disabled' | 'no-tokens'
+
+export type SearchTokenLookupResult =
+  | { matched: true; ids: string[] }
+  | { matched: false; reason: SearchTokenLookupSkipReason }
+
+export type FindEntityIdsBySearchTokensInput = {
+  db: Kysely<SearchTokenDatabase>
+  entityType: string
+  query: string
+  fields?: readonly string[] | null
+  scope?: SearchTokenScope
+  config?: SearchConfig
+}
+
+/**
+ * Resolve the record ids whose indexed `search_tokens` cover every token in
+ * `query`.
+ *
+ * This is the encryption-safe replacement for `ilike` filtering on columns an
+ * encryption map covers: the stored column holds ciphertext, so
+ * `ilike '%term%'` silently matches nothing, while the token index stores
+ * hashes of the plaintext and keeps matching. See issue #2990.
+ *
+ * Matching requires ALL query tokens to be present on the record. The token
+ * search strategy in `@open-mercato/search` uses a looser match ratio; list
+ * endpoints want the stricter behavior so a two-word query narrows rather than
+ * widens the result set.
+ */
+export async function findEntityIdsBySearchTokens({
+  db,
+  entityType,
+  query,
+  fields,
+  scope,
+  config,
+}: FindEntityIdsBySearchTokensInput): Promise<SearchTokenLookupResult> {
+  const trimmed = query.trim()
+  if (!trimmed) return { matched: false, reason: 'empty-query' }
+
+  const searchConfig = config ?? resolveSearchConfig()
+  if (!searchConfig.enabled) return { matched: false, reason: 'search-disabled' }
+
+  const { hashes } = tokenizeText(trimmed, searchConfig)
+  if (!hashes.length) return { matched: false, reason: 'no-tokens' }
+
+  let builder = db
+    .selectFrom('search_tokens')
+    .select('entity_id')
+    .where('entity_type', '=', entityType)
+    .where('token_hash', 'in', hashes)
+
+  const scopedFields = (fields ?? []).filter((field) => typeof field === 'string' && field.length > 0)
+  if (scopedFields.length === 1) {
+    builder = builder.where('field', '=', scopedFields[0])
+  } else if (scopedFields.length > 1) {
+    builder = builder.where('field', 'in', Array.from(scopedFields))
+  }
+
+  if (scope?.tenantId !== undefined) {
+    builder = builder.where(sql<boolean>`tenant_id is not distinct from ${scope.tenantId}`)
+  }
+
+  if (scope?.organizationId !== undefined) {
+    builder = scope.organizationId === null
+      ? builder.where(sql<boolean>`organization_id is not distinct from ${null}`)
+      : builder.where('organization_id', '=', scope.organizationId)
+  } else if (scope?.organizationIds?.length) {
+    builder = builder.where('organization_id', 'in', Array.from(scope.organizationIds))
+  }
+
+  const rows = (await builder
+    .groupBy('entity_id')
+    .having(sql<boolean>`count(distinct token_hash) >= ${hashes.length}`)
+    .execute()) as Array<{ entity_id?: unknown }>
+
+  const ids = rows
+    .map((row) => (typeof row.entity_id === 'string' ? row.entity_id : null))
+    .filter((id): id is string => typeof id === 'string' && id.length > 0)
+
+  return { matched: true, ids }
+}
+
+/**
+ * Legacy-shaped adapter for call sites that predate
+ * {@link SearchTokenLookupResult}: `null` for a blank query, `[]` for every
+ * other non-answer, otherwise the matched ids.
+ *
+ * Prefer {@link findEntityIdsBySearchTokens} in new code — the discriminated
+ * result distinguishes "the index says nothing matched" from "the index was
+ * never consulted", and that distinction is exactly what a `null`/`[]` pair
+ * loses.
+ */
+export async function findEntityIdsBySearchTokensCompat(
+  input: FindEntityIdsBySearchTokensInput,
+): Promise<string[] | null> {
+  const result = await findEntityIdsBySearchTokens(input)
+  if (result.matched) return result.ids
+  return result.reason === 'empty-query' ? null : []
+}

package/src/lib/search/tokenize.ts

@@ -1,5 +1,5 @@
 import crypto from 'crypto'
-import { resolveSearchConfig, type SearchConfig } from './config'
+import { resolveSearchConfig, resolveSearchTokenLimits, type SearchConfig } from './config'
 
 export type TokenizationResult = {
   tokens: string[]
@@ -20,13 +20,28 @@ function splitTokens(text: string, minLength: number): string[] {
     .filter((token) => token.length >= minLength)
 }
 
-function expandToken(token: string, config: SearchConfig): string[] {
-  if (!config.enablePartials) return [token]
-  const results: string[] = []
-  for (let i = config.minTokenLength; i <= token.length; i += 1) {
-    results.push(token.slice(0, i))
+function appendExpandedToken(
+  token: string,
+  config: SearchConfig,
+  seen: Set<string>,
+  tokens: string[],
+  limit: number,
+): void {
+  const append = (candidate: string): boolean => {
+    if (seen.has(candidate)) return tokens.length < limit
+    seen.add(candidate)
+    tokens.push(candidate)
+    return tokens.length < limit
+  }
+
+  if (!config.enablePartials) {
+    append(token)
+    return
+  }
+
+  for (let length = config.minTokenLength; length <= token.length; length += 1) {
+    if (!append(token.slice(0, length))) return
   }
-  return results
 }
 
 export function hashToken(token: string, config?: SearchConfig): string {
@@ -36,10 +51,17 @@ export function hashToken(token: string, config?: SearchConfig): string {
 
 export function tokenizeText(text: string, config?: SearchConfig): TokenizationResult {
   const cfg = config ?? resolveSearchConfig()
-  const baseTokens = splitTokens(text, cfg.minTokenLength)
-  const expanded = baseTokens.flatMap((token) => expandToken(token, cfg))
-  const unique = Array.from(new Set(expanded))
-  const tokens = unique.filter((token) => token.length >= cfg.minTokenLength)
+  const limits = resolveSearchTokenLimits(cfg)
+  const boundedText = limits.maxFieldChars > 0 ? text.slice(0, limits.maxFieldChars) : text
+  const tokenLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY
+  const seen = new Set<string>()
+  const tokens: string[] = []
+
+  for (const token of splitTokens(boundedText, cfg.minTokenLength)) {
+    if (tokens.length >= tokenLimit) break
+    appendExpandedToken(token, cfg, seen, tokens, tokenLimit)
+  }
+
   const hashes = tokens.map((token) => hashToken(token, cfg))
   return { tokens, hashes }
 }

package/src/lib/telemetry/runtime.ts

@@ -0,0 +1,72 @@
+export type TelemetryTraceCarrier = Record<string, string>
+
+export type TelemetryRuntime = {
+  /**
+   * True only when the active SDK may safely use the process-global W3C
+   * propagator for cross-boundary extraction.
+   */
+  canUseGlobalTracePropagation(): boolean
+  captureTraceContext(): TelemetryTraceCarrier
+  continueTrace<T>(
+    carrier: TelemetryTraceCarrier | undefined,
+    name: string,
+    fn: () => T,
+    options?: { kind?: 'internal' | 'server' | 'client' | 'producer' | 'consumer' },
+  ): T
+  recordHttpDuration(method: string, route: string, status: number, startedAt: number): void
+  reportError(
+    error: unknown,
+    context?: {
+      module?: string
+      attributes?: Record<string, string | number | boolean | undefined>
+    },
+  ): void
+  shutdown(): Promise<void>
+}
+
+const GLOBAL_KEY = Symbol.for('@open-mercato/shared.telemetryRuntime')
+const ENABLED_BACKENDS = new Set(['console', 'signoz', 'newrelic', 'otlp'])
+
+type TelemetryRuntimeStore = {
+  active?: TelemetryRuntime
+}
+
+function store(): TelemetryRuntimeStore {
+  const globalStore = globalThis as unknown as Record<symbol, TelemetryRuntimeStore | undefined>
+  let current = globalStore[GLOBAL_KEY]
+  if (!current) {
+    current = {}
+    globalStore[GLOBAL_KEY] = current
+  }
+  return current
+}
+
+/**
+ * This check is intentionally owned by shared code so hosts can decide whether
+ * to dynamically import the telemetry package without evaluating that package.
+ */
+export function isTelemetryBackendEnabled(raw?: string): boolean {
+  const value = raw ?? (
+    typeof process === 'undefined'
+      ? undefined
+      : process.env.TELEMETRY_BACKEND
+  )
+  return ENABLED_BACKENDS.has((value ?? '').trim().toLowerCase())
+}
+
+export function registerTelemetryRuntime(runtime: TelemetryRuntime): () => void {
+  store().active = runtime
+  return () => {
+    const current = store()
+    if (current.active === runtime) current.active = undefined
+  }
+}
+
+export function getTelemetryRuntime(): TelemetryRuntime | undefined {
+  return store().active
+}
+
+/** Test-only: clear the process-wide telemetry bridge. */
+export function resetTelemetryRuntime(): void {
+  store().active = undefined
+}