@open-mercato/shared 0.6.7 → 0.6.8-canary.320.1.bf889f49e4

package/src/lib/query/__tests__/engine.scope-and-or.test.ts

@@ -282,6 +282,73 @@ describe('BasicQueryEngine — null equality', () => {
   })
 })
 
+describe('BasicQueryEngine — doc-field null equality (issue #4841)', () => {
+  function serializeNode(value: any): string {
+    return JSON.stringify(value, (_key, inner) =>
+      inner && typeof inner.toOperationNode === 'function' ? inner.toOperationNode() : inner,
+    )
+  }
+
+  // The doc-backed predicates land inside the entity_indexes EXISTS sub-builder;
+  // pick out the one that reads the JSONB doc (`doc ->> 'field'`).
+  function docPredicates(fakeDb: any, table: string): string[] {
+    const baseCall = fakeDb._calls.find((b: any) => b._ops.table === table)
+    expect(baseCall).toBeTruthy()
+    const existsEntries = (baseCall._ops.wheres as any[]).filter(
+      (where: any) => Array.isArray(where) && where[0] === 'exists',
+    )
+    expect(existsEntries.length).toBeGreaterThan(0)
+    return existsEntries
+      .flatMap((entry: any) => (entry[1]?._ops?.wheres ?? []) as any[])
+      .map((where: any) => serializeNode(where))
+      .filter((serialized: string) => serialized.includes('->>'))
+  }
+
+  async function runDocFilter(filters: Record<string, unknown>) {
+    const fakeDb = createFakeKysely({
+      scheduled_jobs: [],
+      // `ended_at` is deliberately absent, so the field resolves to the
+      // entity_indexes doc rather than to a base column.
+      'information_schema.columns': [
+        { table_name: 'scheduled_jobs', column_name: 'id' },
+      ],
+      'information_schema.tables': [{ table_name: 'entity_indexes' }],
+    })
+    const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
+    await engine.query('scheduler:scheduled_job', {
+      tenantId: 't1',
+      fields: ['id'],
+      omitAutomaticTenantOrgScope: true,
+      filters,
+    })
+    return docPredicates(fakeDb, 'scheduled_jobs')
+  }
+
+  test('$eq null on a doc field compiles to "is null", never "= null"', async () => {
+    const predicates = await runDocFilter({ ended_at: { $eq: null } })
+
+    expect(predicates.length).toBeGreaterThan(0)
+    expect(predicates.some((sql: string) => sql.includes('is null'))).toBe(true)
+    // `(doc ->> 'ended_at') = NULL` is never TRUE in SQL, so it silently
+    // returns zero rows instead of matching unset values.
+    expect(predicates.some((sql: string) => / = /.test(sql))).toBe(false)
+  })
+
+  test('bare null on a doc field compiles to "is null"', async () => {
+    const predicates = await runDocFilter({ ended_at: null })
+
+    expect(predicates.some((sql: string) => sql.includes('is null'))).toBe(true)
+    expect(predicates.some((sql: string) => / = /.test(sql))).toBe(false)
+  })
+
+  test('$ne null on a doc field compiles to "is not null", never "<> null"', async () => {
+    const predicates = await runDocFilter({ started_at: { $ne: null } })
+
+    expect(predicates.some((sql: string) => sql.includes('is not null'))).toBe(true)
+    expect(predicates.some((sql: string) => sql.includes('<>'))).toBe(false)
+  })
+})
+
 describe('BasicQueryEngine — omitAutomaticTenantOrgScope', () => {
   test('skips automatic tenant and organization guards when flag is set', async () => {
     const fakeDb = createFakeKysely({

package/src/lib/query/__tests__/engine.test.ts

@@ -1,10 +1,28 @@
 import { BasicQueryEngine } from '../engine'
 import { SortDir } from '../types'
 import { registerModules } from '../../i18n/server'
+import { clearSearchTokenPresenceCache } from '../../search/availability'
+
+// The token-presence answer is cached process-wide (TTL); without clearing it,
+// probe-count assertions would observe hits from earlier tests in this file.
+beforeEach(() => {
+  clearSearchTokenPresenceCache()
+})
 
 // Mock modules with one entity extension
 const mockModules = [
   { id: 'auth', entityExtensions: [ { base: 'auth:user', extension: 'my_module:user_profile', join: { baseKey: 'id', extensionKey: 'user_id' } } ] },
+  {
+    id: 'example',
+    entityExtensions: [
+      // Declares `table` explicitly; the derived plural now agrees with it.
+      { base: 'customers:customer_entity', extension: 'example:example_customer_priority', join: { baseKey: 'id', extensionKey: 'customer_id' }, table: 'example_customer_priorities' },
+      // No `table`: exercises the derived-plural fallback.
+      { base: 'auth:role', extension: 'example:example_role_policy', join: { baseKey: 'id', extensionKey: 'role_id' } },
+      // `table` is not a bare identifier, so the engine must refuse it.
+      { base: 'auth:session', extension: 'example:example_session_note', join: { baseKey: 'id', extensionKey: 'session_id' }, table: 'notes"; drop table users --' },
+    ],
+  },
 ]
 
 // Register modules for the registration-based pattern
@@ -355,6 +373,8 @@ describe('BasicQueryEngine (Kysely)', () => {
     const fakeDb = createFakeKysely({
       customer_entities: [],
       customer_people: [],
+      search_tokens: [{ one: 1 }],
+      'information_schema.tables': [{ table_name: 'search_tokens' }],
       'information_schema.columns': [
         { table_name: 'customer_entities', column_name: 'tenant_id' },
         { table_name: 'customer_people', column_name: 'id' },
@@ -362,8 +382,6 @@ describe('BasicQueryEngine (Kysely)', () => {
       ],
     })
     const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
-    jest.spyOn(engine as any, 'tableExists').mockResolvedValue(true)
-    jest.spyOn(engine as any, 'hasSearchTokens').mockResolvedValue(true)
     const applySearchTokensSpy = jest.spyOn(engine as any, 'applySearchTokens')
 
     await engine.query('customers:customer_entity', {
@@ -409,7 +427,6 @@ describe('BasicQueryEngine (Kysely)', () => {
       ],
     })
     const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
-    jest.spyOn(engine as any, 'tableExists').mockResolvedValue(false)
     const applySearchTokensSpy = jest.spyOn(engine as any, 'applySearchTokens')
 
     await engine.query('customers:customer_entity', {
@@ -445,10 +462,10 @@ describe('BasicQueryEngine (Kysely)', () => {
   test('uses search tokens for index document fields on base entities', async () => {
     const fakeDb = createFakeKysely({
       todos: [],
+      search_tokens: [{ one: 1 }],
+      'information_schema.tables': [{ table_name: 'search_tokens' }],
     })
     const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
-    const tableExistsSpy = jest.spyOn(engine as any, 'tableExists').mockResolvedValue(true)
-    const hasSearchTokensSpy = jest.spyOn(engine as any, 'hasSearchTokens').mockResolvedValue(true)
     const applySearchTokensSpy = jest.spyOn(engine as any, 'applySearchTokens')
 
     await engine.query('example:todo', {
@@ -461,12 +478,15 @@ describe('BasicQueryEngine (Kysely)', () => {
       page: { page: 1, pageSize: 10 },
     })
 
-    expect(tableExistsSpy).toHaveBeenCalledWith('search_tokens')
-    expect(hasSearchTokensSpy).toHaveBeenCalledWith(
-      'example:todo',
-      't1',
-      expect.objectContaining({ ids: ['org1'] }),
-    )
+    const calls = fakeDb._calls as Array<{ _ops: { table: string; wheres: unknown[][] } }>
+    const tableProbe = calls.find((call) =>
+      call._ops.table === 'information_schema.tables' &&
+      call._ops.wheres.some((where) => where[0] === 'table_name' && where[2] === 'search_tokens'))
+    expect(tableProbe).toBeTruthy()
+    const tokenProbe = calls.find((call) =>
+      call._ops.table === 'search_tokens' &&
+      call._ops.wheres.some((where) => where[0] === 'entity_type' && where[2] === 'example:todo'))
+    expect(tokenProbe).toBeTruthy()
     expect(applySearchTokensSpy).toHaveBeenCalledWith(
       expect.anything(),
       expect.objectContaining({
@@ -488,7 +508,6 @@ describe('BasicQueryEngine (Kysely)', () => {
       ],
     })
     const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
-    const hasSearchTokensSpy = jest.spyOn(engine as any, 'hasSearchTokens').mockResolvedValue(true)
     const applySearchTokensSpy = jest.spyOn(engine as any, 'applySearchTokens')
 
     await engine.query('example:todo', {
@@ -501,7 +520,8 @@ describe('BasicQueryEngine (Kysely)', () => {
       page: { page: 1, pageSize: 10 },
     })
 
-    expect(hasSearchTokensSpy).not.toHaveBeenCalled()
+    const calls = fakeDb._calls as Array<{ _ops: { table: string } }>
+    expect(calls.some((call) => call._ops.table === 'search_tokens')).toBe(false)
     expect(applySearchTokensSpy).not.toHaveBeenCalled()
     const baseCall = fakeDb._calls.find((builder: any) => builder._ops.table === 'todos')
     expect(baseCall?._ops.wheres).toContainEqual(['todos.search_text', 'ilike', '%avision%'])
@@ -933,4 +953,184 @@ describe('BasicQueryEngine (Kysely)', () => {
     expect(baseCall._ops.limits).toBe(10)
     expect(baseCall._ops.offsets).toBe(10)
   })
+
+  // A tiebreak sort is only worth configuring if the engine actually emits it.
+  // `list.tiebreakSortField` (used by the sales line routes to keep lines with an
+  // equal `line_number` in a repeatable order) is the first caller to pass more
+  // than one sort element, so pin that every element reaches ORDER BY in order —
+  // dropping sort[1] would silently restore the non-determinism it exists to fix.
+  test('emits every sort element as an ORDER BY column, in order', async () => {
+    const fakeDb = createFakeKysely({
+      sales_order_lines: [
+        { id: 'b', tenant_id: 't1', organization_id: 'org1', line_number: 0 },
+        { id: 'a', tenant_id: 't1', organization_id: 'org1', line_number: 0 },
+      ],
+      'information_schema.columns': [
+        { table_name: 'sales_order_lines', column_name: 'id' },
+        { table_name: 'sales_order_lines', column_name: 'tenant_id' },
+        { table_name: 'sales_order_lines', column_name: 'organization_id' },
+        { table_name: 'sales_order_lines', column_name: 'deleted_at' },
+        { table_name: 'sales_order_lines', column_name: 'line_number' },
+      ],
+    })
+    const engine = new BasicQueryEngine(
+      {} as any,
+      () => fakeDb as any,
+      () => ({
+        isEnabled: () => true,
+        getEncryptedFieldNames: async () => [],
+      }),
+    )
+
+    await engine.query('sales:sales_order_line', {
+      tenantId: 't1',
+      organizationId: 'org1',
+      fields: ['id', 'line_number'],
+      sort: [
+        { field: 'line_number', dir: SortDir.Asc },
+        { field: 'id', dir: SortDir.Asc },
+      ],
+      page: { page: 1, pageSize: 10 },
+    })
+
+    const baseCall = fakeDb._calls.find((call: any) => call._ops.table === 'sales_order_lines')
+    expect(baseCall._ops.orderBys).toEqual([
+      ['sales_order_lines.line_number', 'asc'],
+      ['sales_order_lines.id', 'asc'],
+    ])
+  })
+
+  test('keeps each sort element on its own direction', async () => {
+    const fakeDb = createFakeKysely({
+      sales_order_lines: [
+        { id: 'a', tenant_id: 't1', organization_id: 'org1', line_number: 1 },
+      ],
+      'information_schema.columns': [
+        { table_name: 'sales_order_lines', column_name: 'id' },
+        { table_name: 'sales_order_lines', column_name: 'tenant_id' },
+        { table_name: 'sales_order_lines', column_name: 'organization_id' },
+        { table_name: 'sales_order_lines', column_name: 'deleted_at' },
+        { table_name: 'sales_order_lines', column_name: 'line_number' },
+      ],
+    })
+    const engine = new BasicQueryEngine(
+      {} as any,
+      () => fakeDb as any,
+      () => ({
+        isEnabled: () => true,
+        getEncryptedFieldNames: async () => [],
+      }),
+    )
+
+    await engine.query('sales:sales_order_line', {
+      tenantId: 't1',
+      organizationId: 'org1',
+      fields: ['id', 'line_number'],
+      sort: [
+        { field: 'line_number', dir: SortDir.Desc },
+        { field: 'id', dir: SortDir.Asc },
+      ],
+      page: { page: 1, pageSize: 10 },
+    })
+
+    const baseCall = fakeDb._calls.find((call: any) => call._ops.table === 'sales_order_lines')
+    expect(baseCall._ops.orderBys).toEqual([
+      ['sales_order_lines.line_number', 'desc'],
+      ['sales_order_lines.id', 'asc'],
+    ])
+  })
+
+  describe('search_tokens coverage probe (#4723 parity)', () => {
+    type ProbeDbLog = { _calls: Array<{ _ops: { table: string } }> }
+
+    const countProbes = (fakeDb: ProbeDbLog): number =>
+      fakeDb._calls.filter((call) => call._ops.table === 'search_tokens').length
+
+    const buildEngine = (fakeDb: unknown): BasicQueryEngine => new BasicQueryEngine(
+      {} as ConstructorParameters<typeof BasicQueryEngine>[0],
+      (() => fakeDb) as unknown as NonNullable<ConstructorParameters<typeof BasicQueryEngine>[1]>,
+    )
+
+    const buildDb = () => createFakeKysely({
+      users: [],
+      'information_schema.tables': [{ table_name: 'search_tokens' }],
+    })
+
+    test('is skipped when the query carries no like/ilike filter', async () => {
+      const fakeDb = buildDb()
+      const engine = buildEngine(fakeDb)
+
+      await engine.query('auth:user', {
+        tenantId: 't1',
+        organizationId: 'org1',
+        filters: { is_active: { $eq: true } },
+      })
+
+      expect(countProbes(fakeDb)).toBe(0)
+    })
+
+    test('still runs when the query actually searches', async () => {
+      const fakeDb = buildDb()
+      const engine = buildEngine(fakeDb)
+
+      await engine.query('auth:user', {
+        tenantId: 't1',
+        organizationId: 'org1',
+        filters: { email: { $ilike: '%abc%' } },
+      })
+
+      expect(countProbes(fakeDb)).toBeGreaterThan(0)
+    })
+  })
+})
+
+describe('BasicQueryEngine entity-extension joins', () => {
+  function extensionJoins(fakeDb: any, baseTable: string): any[] {
+    const baseCall = fakeDb._calls.find((builder: any) => builder._ops.table === baseTable)
+    expect(baseCall).toBeTruthy()
+    return baseCall._ops.joins.filter((entry: any) => Object.keys(entry.aliasObj)[0].startsWith('ext_'))
+  }
+
+  async function joinFor(entity: string, baseTable: string): Promise<any> {
+    const fakeDb = createFakeKysely()
+    const engine = new BasicQueryEngine({} as any, () => fakeDb as any)
+    await engine.query(entity, {
+      tenantId: 't1',
+      organizationId: 'org1',
+      fields: ['id'],
+      includeExtensions: true,
+    })
+    const joins = extensionJoins(fakeDb, baseTable)
+    expect(joins).toHaveLength(1)
+    return joins[0]
+  }
+
+  test('prefers a declared table over the derived plural', async () => {
+    const join = await joinFor('customers:customer_entity', 'customer_entities')
+    expect(join.aliasObj).toEqual({ ext_example_customer_priority: 'example_customer_priorities' })
+    expect(join.conditions).toEqual([
+      {
+        method: 'on',
+        args: ['ext_example_customer_priority.customer_id', '=', 'customer_entities.id'],
+      },
+    ])
+  })
+
+  test('falls back to the derived plural when no table is declared', async () => {
+    // `policy` ends in `y`, so the correct plural is `policies`. This previously asserted
+    // `policys`, pinning a separate inline `+s` pluralizer that the extension-join path used
+    // instead of the file's own `pluralizeBaseName` — the very bug that made
+    // `example_customer_priority` derive `example_customer_prioritys` and forced the
+    // `table` override into existence.
+    const join = await joinFor('auth:role', 'roles')
+    expect(join.aliasObj).toEqual({ ext_example_role_policy: 'example_role_policies' })
+    expect(join.conditions).toEqual([
+      { method: 'on', args: ['ext_example_role_policy.role_id', '=', 'roles.id'] },
+    ])
+  })
+
+  test('ignores a declared table that is not a bare identifier', async () => {
+    const join = await joinFor('auth:session', 'sessions')
+    expect(join.aliasObj).toEqual({ ext_example_session_note: 'example_session_notes' })
+  })
 })

package/src/lib/query/ciphertext-search-warning.ts

@@ -0,0 +1,106 @@
+import { createLogger } from '../logger'
+import type { EntityId } from '../../modules/entities'
+
+const logger = createLogger('shared').child({ component: 'query' })
+
+/**
+ * Minimal slice of `TenantDataEncryptionService` needed to decide whether a
+ * text filter targets an encrypted column.
+ */
+export type CiphertextWarningEncryptionService = {
+  getEncryptedFieldNames?: (
+    entityId: EntityId,
+    tenantId?: string | null,
+    organizationId?: string | null,
+  ) => Promise<readonly string[]>
+  isEnabled?: () => boolean
+} | null | undefined
+
+export type CiphertextLikeFallbackReason =
+  | 'no-indexable-tokens'
+  | 'no-search-tokens'
+  | 'search-disabled'
+  | 'raw-orm-filter'
+
+const REASON_HINTS: Record<CiphertextLikeFallbackReason, string> = {
+  'search-disabled':
+    'OM_SEARCH_ENABLED is off, so the filter runs as ILIKE against ciphertext and matches nothing.',
+  'no-indexable-tokens':
+    'The filter value produced no search tokens, so it runs as ILIKE against ciphertext and matches nothing. Use at least OM_SEARCH_MIN_LEN indexable characters, or filter on the deterministic hash column.',
+  'no-search-tokens':
+    'No search_tokens exist for this entity and scope, so the filter runs as ILIKE against ciphertext and matches nothing. Index the entity through query_index and reindex, or filter on the deterministic hash column.',
+  'raw-orm-filter':
+    'This filter bypasses the query engine, so it runs as ILIKE against ciphertext and matches nothing. Resolve ids through findEntityIdsBySearchTokens (@open-mercato/shared/lib/search/tokenLookup), filter on the deterministic hash column, or compare in memory after decryption.',
+}
+
+// One warning per entity/tenant/field per process — the fallback repeats on
+// every request, and an operator only needs to learn about it once.
+const WARN_CACHE_CAP = 5000
+const warned = new Set<string>()
+
+const warnKey = (entity: string, tenantId: string | null, field: string): string =>
+  `${entity}|${tenantId ?? 'null'}|${field}`
+
+export const normalizeColumnName = (value: string): string =>
+  value.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()
+
+/** Test seam: the warn-once cache is process-global by design. */
+export function resetCiphertextLikeWarnCache(): void {
+  warned.clear()
+}
+
+/**
+ * Warn when a `like`/`ilike` filter is about to run against a column that an
+ * encryption map covers while the token index is unavailable.
+ *
+ * Without this the query degrades silently: the predicate compares a plaintext
+ * pattern against ciphertext, matches nothing, and the endpoint returns an
+ * empty page that is indistinguishable from a genuine no-result. Issue #2990.
+ *
+ * Never throws and never blocks the query — a failure here is logged at debug
+ * level and the caller proceeds unchanged.
+ */
+export async function warnOnCiphertextLikeFallback(params: {
+  entity: string
+  fields: readonly string[]
+  tenantId: string | null
+  reason: CiphertextLikeFallbackReason
+  service: CiphertextWarningEncryptionService
+}): Promise<void> {
+  const { entity, tenantId, reason, service } = params
+  if (!service || service.isEnabled?.() === false) return
+  if (typeof service.getEncryptedFieldNames !== 'function') return
+
+  const candidates = params.fields
+    .filter((field) => typeof field === 'string' && !field.startsWith('cf:'))
+    .map((field) => normalizeColumnName(field))
+    .filter((field, index, all) => field.length > 0 && all.indexOf(field) === index)
+    .filter((field) => !warned.has(warnKey(entity, tenantId, field)))
+  if (!candidates.length) return
+
+  try {
+    // `organizationId: null` resolves the global map plus every per-organization
+    // override, so a field encrypted for any organization is reported.
+    const encrypted = new Set(
+      (await service.getEncryptedFieldNames(entity, tenantId, null)).map((field) =>
+        normalizeColumnName(String(field)),
+      ),
+    )
+    for (const field of candidates) {
+      if (!encrypted.has(field)) continue
+      if (warned.size >= WARN_CACHE_CAP) warned.clear()
+      warned.add(warnKey(entity, tenantId, field))
+      logger.warn('Text search filter cannot match an encrypted column', {
+        entity,
+        field,
+        reason,
+        hint: REASON_HINTS[reason],
+      })
+    }
+  } catch (err) {
+    logger.debug('Ciphertext search warning check failed', {
+      entity,
+      err: err instanceof Error ? err.message : String(err),
+    })
+  }
+}