@open-mercato/shared 0.6.7-develop.6860.1.f796b22e80 → 0.6.7

package/src/lib/search/availability.ts

@@ -1,232 +0,0 @@
-import { sql } from 'kysely'
-import { parseNumberWithDefault } from '@open-mercato/shared/lib/number'
-
-export type OrganizationScope = { ids: string[]; includeNull: boolean }
-
-export type SearchTokenSourceRef = { entity: string; recordIdColumn?: string }
-
-type ProbeExpression = object | string | readonly string[] | null
-
-export type SearchTokenProbeQueryBuilder = {
-  select: (selection: ProbeExpression) => SearchTokenProbeQueryBuilder
-  where: (column: ProbeExpression, operator?: string, value?: ProbeExpression) => SearchTokenProbeQueryBuilder
-  limit: (count: number) => SearchTokenProbeQueryBuilder
-  executeTakeFirst: () => Promise<object | undefined>
-}
-
-export type SearchTokenProbeDb = { selectFrom: (table: string) => SearchTokenProbeQueryBuilder }
-
-export type SearchTokenAvailabilityDebugPayload = {
-  entity: string
-  tenantId: string | null
-  organizationScope?: OrganizationScope | null
-  recordIdColumn?: string
-  hasTokens?: boolean
-  error?: string
-}
-
-export type SearchTokenAvailabilityDeps = {
-  getDb: () => SearchTokenProbeDb
-  getConfig: () => { enabled: boolean }
-  applyOrganizationScope: (
-    query: SearchTokenProbeQueryBuilder,
-    column: string,
-    scope: OrganizationScope,
-  ) => SearchTokenProbeQueryBuilder
-  logDebug: (event: string, payload: SearchTokenAvailabilityDebugPayload) => void
-}
-
-export type SearchTokenAvailability = {
-  /**
-   * The cheap, statically-known half of the decision: search is configured on
-   * AND the `search_tokens` table exists. Safe to resolve eagerly (consumers
-   * like custom-field source attachment need it before any filter is
-   * inspected); the table probe is memoized per instance.
-   */
-  staticEnabled: () => Promise<boolean>
-  /**
-   * The expensive half: does `search_tokens` hold any row for this
-   * (entity, tenant, organization scope)? Historically the `LIMIT 1` probe
-   * could degenerate into a seq scan on a large table (#4723), so callers MUST
-   * only ask when the query actually carries a like/ilike filter — use
-   * `hasSearchFilter` for the gate. Three layers keep it cheap for the end
-   * user: index-usable predicates (no `IS NOT DISTINCT FROM`) served by the
-   * dedicated `search_tokens_presence_idx (entity_type, tenant_id,
-   * organization_id)` prefix — which makes the MISS as cheap as the hit — a
-   * per-request instance memo, and a process-level TTL cache
-   * (`OM_SEARCH_TOKEN_PRESENCE_CACHE_MS`, default 30s) that amortizes the
-   * probe across requests. Probe errors log `search:has-tokens-error`,
-   * resolve to `false`, and are never TTL-cached.
-   */
-  hasTokens: (entity: string, tenantId: string | null, orgScope?: OrganizationScope | null) => Promise<boolean>
-  /** First-hit sweep over token sources; logs `search:source-has-tokens` per probed source. */
-  anySourceHasTokens: (
-    sources: SearchTokenSourceRef[],
-    tenantId: string | null,
-    orgScope?: OrganizationScope | null,
-  ) => Promise<boolean>
-}
-
-export function isSearchFilterOp(op: string | null | undefined): boolean {
-  return op === 'like' || op === 'ilike'
-}
-
-/**
- * The single definition of "this query actually searches". Every consumer of
- * the token-availability answer sits behind a like/ilike guard, so when this
- * returns `false` the `hasTokens` probe's answer would never be read — gate
- * the probe on it.
- */
-export function hasSearchFilter(filters: ReadonlyArray<{ op?: string | null }>): boolean {
-  return filters.some((filter) => isSearchFilterOp(filter.op))
-}
-
-function orgScopeKey(scope: OrganizationScope | null | undefined): string {
-  if (!scope) return 'none'
-  return `${scope.includeNull ? '1' : '0'}:${[...scope.ids].sort((left, right) => left.localeCompare(right)).join(',')}`
-}
-
-const PRESENCE_CACHE_DEFAULT_TTL_MS = 30_000
-const PRESENCE_CACHE_MAX_ENTRIES = 10_000
-
-// Process-level TTL cache for the token-presence answer. Module-level (process-global) on
-// purpose: `createRequestContainer` builds fresh engines — and with them fresh resolver
-// instances — per request, so an instance-scoped memo alone re-pays the probe on every
-// request. On a large `search_tokens` one probe can be pathologically expensive (#4723),
-// so the answer is amortized across requests here and only re-checked once per TTL.
-// Staleness contract: a stale `false` keeps like/ilike on the plain-column fallback for up
-// to the TTL after an entity's first tokens are written; a stale `true` routes search
-// through an emptied token set for up to the TTL after a purge. Both converge within the
-// TTL; set OM_SEARCH_TOKEN_PRESENCE_CACHE_MS=0 to disable and probe per request again.
-const presenceCache = new Map<string, { value: boolean; expiresAt: number }>()
-
-function resolvePresenceCacheTtlMs(): number {
-  return parseNumberWithDefault(process.env.OM_SEARCH_TOKEN_PRESENCE_CACHE_MS, PRESENCE_CACHE_DEFAULT_TTL_MS, { integer: true, min: 0 })
-}
-
-function storePresence(key: string, value: boolean, ttlMs: number): void {
-  if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) {
-    const now = Date.now()
-    for (const [entryKey, entry] of presenceCache) {
-      if (entry.expiresAt <= now) presenceCache.delete(entryKey)
-    }
-    if (presenceCache.size >= PRESENCE_CACHE_MAX_ENTRIES) presenceCache.clear()
-  }
-  presenceCache.set(key, { value, expiresAt: Date.now() + ttlMs })
-}
-
-export function clearSearchTokenPresenceCache(): void {
-  presenceCache.clear()
-}
-
-/**
- * One place that answers "is token search usable here?" for both query
- * engines, instead of each hand-assembling config + table-existence +
- * token-presence probes with private duplicate helpers and ad-hoc memo maps.
- *
- * Memoization is per instance; engines are constructed per request
- * (`createRequestContainer`), so entries never outlive a request — the same
- * staleness contract the engines' previous per-query join maps had. A
- * rejected table probe is evicted so the next call retries instead of
- * observing a poisoned cache entry.
- */
-export function createSearchTokenAvailability(deps: SearchTokenAvailabilityDeps): SearchTokenAvailability {
-  const tablePresence = new Map<string, Promise<boolean>>()
-  const tokenPresence = new Map<string, Promise<boolean>>()
-
-  const tableExists = (table: string): Promise<boolean> => {
-    const cached = tablePresence.get(table)
-    if (cached) return cached
-    const probe = (async () => {
-      const row = await deps.getDb()
-        .selectFrom('information_schema.tables')
-        .select(sql<number>`1`.as('one'))
-        .where('table_name', '=', table)
-        .limit(1)
-        .executeTakeFirst()
-      return !!row
-    })()
-    tablePresence.set(table, probe)
-    probe.catch(() => tablePresence.delete(table))
-    return probe
-  }
-
-  const probeTokens = async (
-    entity: string,
-    tenantId: string | null,
-    orgScope?: OrganizationScope | null,
-  ): Promise<boolean> => {
-    let query = deps.getDb()
-      .selectFrom('search_tokens')
-      .select(sql<number>`1`.as('one'))
-      .where('entity_type', '=', entity)
-    // Deliberately `= / IS NULL` instead of `IS NOT DISTINCT FROM` (identical semantics
-    // for a string|null tenant): the latter cannot serve as an index condition, which is
-    // part of why the planner degraded this probe to a seq scan on large tables (#4723).
-    // With plain predicates the probe is a pure prefix seek on
-    // `search_tokens_presence_idx (entity_type, tenant_id, organization_id)`, making the
-    // miss as cheap as the hit.
-    query = tenantId == null
-      ? query.where('tenant_id', 'is', null)
-      : query.where('tenant_id', '=', tenantId)
-    if (orgScope) {
-      query = deps.applyOrganizationScope(query, 'search_tokens.organization_id', orgScope)
-    }
-    const row = await query.limit(1).executeTakeFirst()
-    return !!row
-  }
-
-  const hasTokens = (
-    entity: string,
-    tenantId: string | null,
-    orgScope?: OrganizationScope | null,
-  ): Promise<boolean> => {
-    const key = `${entity}|${tenantId ?? '__null__'}|${orgScopeKey(orgScope)}`
-    const ttlMs = resolvePresenceCacheTtlMs()
-    if (ttlMs > 0) {
-      const entry = presenceCache.get(key)
-      if (entry && entry.expiresAt > Date.now()) return Promise.resolve(entry.value)
-    }
-    const cached = tokenPresence.get(key)
-    if (cached) return cached
-    const probe = (async () => {
-      try {
-        const value = await probeTokens(entity, tenantId, orgScope)
-        // Only genuine probe results enter the process-level cache — caching an
-        // error-driven `false` would pin degraded search for a full TTL after a
-        // transient DB failure.
-        if (ttlMs > 0) storePresence(key, value, ttlMs)
-        return value
-      } catch (err) {
-        deps.logDebug('search:has-tokens-error', {
-          entity,
-          tenantId,
-          organizationScope: orgScope,
-          error: err instanceof Error ? err.message : String(err),
-        })
-        return false
-      }
-    })()
-    tokenPresence.set(key, probe)
-    return probe
-  }
-
-  return {
-    staticEnabled: async () => deps.getConfig().enabled && await tableExists('search_tokens'),
-    hasTokens,
-    anySourceHasTokens: async (sources, tenantId, orgScope) => {
-      for (const source of sources) {
-        const ok = await hasTokens(source.entity, tenantId, orgScope)
-        deps.logDebug('search:source-has-tokens', {
-          entity: source.entity,
-          recordIdColumn: source.recordIdColumn,
-          tenantId,
-          organizationScope: orgScope,
-          hasTokens: ok,
-        })
-        if (ok) return true
-      }
-      return false
-    },
-  }
-}

package/src/lib/search/tokenLookup.ts

@@ -1,133 +0,0 @@
-import { type Kysely, sql } from 'kysely'
-import { resolveSearchConfig, type SearchConfig } from './config'
-import { tokenizeText } from './tokenize'
-
-export type SearchTokenDatabase = {
-  search_tokens: {
-    entity_id: string
-    entity_type: string
-    field: string
-    token_hash: string
-    tenant_id: string | null
-    organization_id: string | null
-  }
-}
-
-/**
- * Tenant/organization scoping for a `search_tokens` lookup.
- *
- * `undefined` and `null` are NOT interchangeable:
- * - `undefined` omits the predicate entirely (the caller owns visibility).
- * - `null` emits a null-safe predicate that matches only globally scoped rows.
- */
-export type SearchTokenScope = {
-  tenantId?: string | null
-  organizationId?: string | null
-  organizationIds?: readonly string[] | null
-}
-
-/**
- * Why a lookup could not produce an id set. Callers MUST NOT read these as
- * "nothing matched" — the token index was never consulted, so the caller's own
- * predicate (usually an `ilike`) is still the authoritative one.
- */
-export type SearchTokenLookupSkipReason = 'empty-query' | 'search-disabled' | 'no-tokens'
-
-export type SearchTokenLookupResult =
-  | { matched: true; ids: string[] }
-  | { matched: false; reason: SearchTokenLookupSkipReason }
-
-export type FindEntityIdsBySearchTokensInput = {
-  db: Kysely<SearchTokenDatabase>
-  entityType: string
-  query: string
-  fields?: readonly string[] | null
-  scope?: SearchTokenScope
-  config?: SearchConfig
-}
-
-/**
- * Resolve the record ids whose indexed `search_tokens` cover every token in
- * `query`.
- *
- * This is the encryption-safe replacement for `ilike` filtering on columns an
- * encryption map covers: the stored column holds ciphertext, so
- * `ilike '%term%'` silently matches nothing, while the token index stores
- * hashes of the plaintext and keeps matching. See issue #2990.
- *
- * Matching requires ALL query tokens to be present on the record. The token
- * search strategy in `@open-mercato/search` uses a looser match ratio; list
- * endpoints want the stricter behavior so a two-word query narrows rather than
- * widens the result set.
- */
-export async function findEntityIdsBySearchTokens({
-  db,
-  entityType,
-  query,
-  fields,
-  scope,
-  config,
-}: FindEntityIdsBySearchTokensInput): Promise<SearchTokenLookupResult> {
-  const trimmed = query.trim()
-  if (!trimmed) return { matched: false, reason: 'empty-query' }
-
-  const searchConfig = config ?? resolveSearchConfig()
-  if (!searchConfig.enabled) return { matched: false, reason: 'search-disabled' }
-
-  const { hashes } = tokenizeText(trimmed, searchConfig)
-  if (!hashes.length) return { matched: false, reason: 'no-tokens' }
-
-  let builder = db
-    .selectFrom('search_tokens')
-    .select('entity_id')
-    .where('entity_type', '=', entityType)
-    .where('token_hash', 'in', hashes)
-
-  const scopedFields = (fields ?? []).filter((field) => typeof field === 'string' && field.length > 0)
-  if (scopedFields.length === 1) {
-    builder = builder.where('field', '=', scopedFields[0])
-  } else if (scopedFields.length > 1) {
-    builder = builder.where('field', 'in', Array.from(scopedFields))
-  }
-
-  if (scope?.tenantId !== undefined) {
-    builder = builder.where(sql<boolean>`tenant_id is not distinct from ${scope.tenantId}`)
-  }
-
-  if (scope?.organizationId !== undefined) {
-    builder = scope.organizationId === null
-      ? builder.where(sql<boolean>`organization_id is not distinct from ${null}`)
-      : builder.where('organization_id', '=', scope.organizationId)
-  } else if (scope?.organizationIds?.length) {
-    builder = builder.where('organization_id', 'in', Array.from(scope.organizationIds))
-  }
-
-  const rows = (await builder
-    .groupBy('entity_id')
-    .having(sql<boolean>`count(distinct token_hash) >= ${hashes.length}`)
-    .execute()) as Array<{ entity_id?: unknown }>
-
-  const ids = rows
-    .map((row) => (typeof row.entity_id === 'string' ? row.entity_id : null))
-    .filter((id): id is string => typeof id === 'string' && id.length > 0)
-
-  return { matched: true, ids }
-}
-
-/**
- * Legacy-shaped adapter for call sites that predate
- * {@link SearchTokenLookupResult}: `null` for a blank query, `[]` for every
- * other non-answer, otherwise the matched ids.
- *
- * Prefer {@link findEntityIdsBySearchTokens} in new code — the discriminated
- * result distinguishes "the index says nothing matched" from "the index was
- * never consulted", and that distinction is exactly what a `null`/`[]` pair
- * loses.
- */
-export async function findEntityIdsBySearchTokensCompat(
-  input: FindEntityIdsBySearchTokensInput,
-): Promise<string[] | null> {
-  const result = await findEntityIdsBySearchTokens(input)
-  if (result.matched) return result.ids
-  return result.reason === 'empty-query' ? null : []
-}

package/src/lib/telemetry/runtime.ts

@@ -1,72 +0,0 @@
-export type TelemetryTraceCarrier = Record<string, string>
-
-export type TelemetryRuntime = {
-  /**
-   * True only when the active SDK may safely use the process-global W3C
-   * propagator for cross-boundary extraction.
-   */
-  canUseGlobalTracePropagation(): boolean
-  captureTraceContext(): TelemetryTraceCarrier
-  continueTrace<T>(
-    carrier: TelemetryTraceCarrier | undefined,
-    name: string,
-    fn: () => T,
-    options?: { kind?: 'internal' | 'server' | 'client' | 'producer' | 'consumer' },
-  ): T
-  recordHttpDuration(method: string, route: string, status: number, startedAt: number): void
-  reportError(
-    error: unknown,
-    context?: {
-      module?: string
-      attributes?: Record<string, string | number | boolean | undefined>
-    },
-  ): void
-  shutdown(): Promise<void>
-}
-
-const GLOBAL_KEY = Symbol.for('@open-mercato/shared.telemetryRuntime')
-const ENABLED_BACKENDS = new Set(['console', 'signoz', 'newrelic', 'otlp'])
-
-type TelemetryRuntimeStore = {
-  active?: TelemetryRuntime
-}
-
-function store(): TelemetryRuntimeStore {
-  const globalStore = globalThis as unknown as Record<symbol, TelemetryRuntimeStore | undefined>
-  let current = globalStore[GLOBAL_KEY]
-  if (!current) {
-    current = {}
-    globalStore[GLOBAL_KEY] = current
-  }
-  return current
-}
-
-/**
- * This check is intentionally owned by shared code so hosts can decide whether
- * to dynamically import the telemetry package without evaluating that package.
- */
-export function isTelemetryBackendEnabled(raw?: string): boolean {
-  const value = raw ?? (
-    typeof process === 'undefined'
-      ? undefined
-      : process.env.TELEMETRY_BACKEND
-  )
-  return ENABLED_BACKENDS.has((value ?? '').trim().toLowerCase())
-}
-
-export function registerTelemetryRuntime(runtime: TelemetryRuntime): () => void {
-  store().active = runtime
-  return () => {
-    const current = store()
-    if (current.active === runtime) current.active = undefined
-  }
-}
-
-export function getTelemetryRuntime(): TelemetryRuntime | undefined {
-  return store().active
-}
-
-/** Test-only: clear the process-wide telemetry bridge. */
-export function resetTelemetryRuntime(): void {
-  store().active = undefined
-}

package/src/lib/webhooks/__tests__/body.test.ts

@@ -1,95 +0,0 @@
-/** @jest-environment node */
-import {
-  DEFAULT_WEBHOOK_BODY_LIMIT_BYTES,
-  readBoundedRequestBody,
-  resolveWebhookBodyLimitBytes,
-  WebhookBodyTooLargeError,
-} from '../body'
-
-function makeStreamingRequest(chunks: Uint8Array[], headers?: HeadersInit) {
-  let index = 0
-  const cancel = jest.fn()
-  const body = new ReadableStream<Uint8Array>({
-    pull(controller) {
-      const chunk = chunks[index]
-      index += 1
-      if (chunk) controller.enqueue(chunk)
-      else controller.close()
-    },
-    cancel,
-  })
-  const request = new Request('http://localhost/webhook', {
-    method: 'POST',
-    headers,
-    body,
-    duplex: 'half',
-  } as RequestInit & { duplex: 'half' })
-  return { request, cancel }
-}
-
-describe('readBoundedRequestBody', () => {
-  const encoder = new TextEncoder()
-
-  it('rejects an oversized declared length before reading the body', async () => {
-    const request = new Request('http://localhost/webhook', {
-      method: 'POST',
-      headers: { 'content-length': '6' },
-      body: 'ok',
-    })
-
-    await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toEqual(
-      expect.objectContaining<WebhookBodyTooLargeError>({ limitBytes: 5 }),
-    )
-  })
-
-  it('stops a chunked body when its actual byte count crosses the limit', async () => {
-    const { request, cancel } = makeStreamingRequest([
-      encoder.encode('abc'),
-      encoder.encode('def'),
-      encoder.encode('never-read'),
-    ])
-
-    await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toBeInstanceOf(
-      WebhookBodyTooLargeError,
-    )
-    expect(cancel).toHaveBeenCalled()
-  })
-
-  it.each([
-    ['invalid', 'not-a-number'],
-    ['lying', '3'],
-  ])('does not let a %s Content-Length bypass the streamed cap', async (_case, contentLength) => {
-    const { request } = makeStreamingRequest(
-      [encoder.encode('abc'), encoder.encode('def')],
-      { 'content-length': contentLength },
-    )
-
-    await expect(readBoundedRequestBody(request, { maxBytes: 5 })).rejects.toBeInstanceOf(
-      WebhookBodyTooLargeError,
-    )
-  })
-
-  it('preserves the exact decoded body at the byte boundary', async () => {
-    const bytes = encoder.encode('a€')
-    const { request } = makeStreamingRequest([bytes.slice(0, 2), bytes.slice(2)])
-
-    await expect(readBoundedRequestBody(request, { maxBytes: bytes.byteLength })).resolves.toBe('a€')
-  })
-})
-
-describe('resolveWebhookBodyLimitBytes', () => {
-  it('uses the documented default for missing or invalid configuration', () => {
-    expect(resolveWebhookBodyLimitBytes(undefined)).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
-    expect(resolveWebhookBodyLimitBytes('invalid')).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
-    expect(resolveWebhookBodyLimitBytes('0')).toBe(DEFAULT_WEBHOOK_BODY_LIMIT_BYTES)
-  })
-
-  it('accepts a positive safe integer override', () => {
-    expect(resolveWebhookBodyLimitBytes('2097152')).toBe(2 * 1024 * 1024)
-  })
-
-  it('supports a source-specific fallback without weakening validation', () => {
-    expect(resolveWebhookBodyLimitBytes(undefined, 2 * 1024 * 1024)).toBe(2 * 1024 * 1024)
-    expect(resolveWebhookBodyLimitBytes('invalid', 2 * 1024 * 1024)).toBe(2 * 1024 * 1024)
-  })
-})

package/src/lib/webhooks/body.ts

@@ -1,70 +0,0 @@
-import { parseNumberWithDefault } from '../number'
-
-export const DEFAULT_WEBHOOK_BODY_LIMIT_BYTES = 1024 * 1024
-
-export class WebhookBodyTooLargeError extends Error {
-  readonly limitBytes: number
-
-  constructor(limitBytes: number) {
-    super(`Webhook body exceeds the ${limitBytes}-byte limit`)
-    this.name = 'WebhookBodyTooLargeError'
-    this.limitBytes = limitBytes
-  }
-}
-
-export function resolveWebhookBodyLimitBytes(
-  raw = typeof process === 'undefined' ? undefined : process.env.OM_WEBHOOK_MAX_BODY_BYTES,
-  fallbackBytes = DEFAULT_WEBHOOK_BODY_LIMIT_BYTES,
-): number {
-  const fallback = Number.isSafeInteger(fallbackBytes) && fallbackBytes > 0
-    ? fallbackBytes
-    : DEFAULT_WEBHOOK_BODY_LIMIT_BYTES
-  const parsed = parseNumberWithDefault(raw, fallback, { min: 1 })
-  return Number.isSafeInteger(parsed) ? parsed : fallback
-}
-
-export async function readBoundedRequestBody(
-  request: Request,
-  options?: { maxBytes?: number },
-): Promise<string> {
-  const configuredLimit = options?.maxBytes ?? resolveWebhookBodyLimitBytes()
-  const maxBytes = Number.isSafeInteger(configuredLimit) && configuredLimit > 0
-    ? configuredLimit
-    : DEFAULT_WEBHOOK_BODY_LIMIT_BYTES
-  const declaredLength = request.headers.get('content-length')?.trim()
-  if (declaredLength && /^\d+$/.test(declaredLength)) {
-    const declaredBytes = Number(declaredLength)
-    if (!Number.isSafeInteger(declaredBytes) || declaredBytes > maxBytes) {
-      throw new WebhookBodyTooLargeError(maxBytes)
-    }
-  }
-
-  if (!request.body) return ''
-
-  const reader = request.body.getReader()
-  const chunks: Uint8Array[] = []
-  let totalBytes = 0
-
-  try {
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      totalBytes += value.byteLength
-      if (totalBytes > maxBytes) {
-        await reader.cancel().catch(() => undefined)
-        throw new WebhookBodyTooLargeError(maxBytes)
-      }
-      chunks.push(value)
-    }
-  } finally {
-    reader.releaseLock()
-  }
-
-  const body = new Uint8Array(totalBytes)
-  let offset = 0
-  for (const chunk of chunks) {
-    body.set(chunk, offset)
-    offset += chunk.byteLength
-  }
-  return new TextDecoder().decode(body)
-}