@open-mercato/shared 0.6.7 → 0.6.8-develop.6874.1.982d6097d8

package/src/lib/search/__tests__/config.test.ts

@@ -1,7 +1,12 @@
 import {
+  DEFAULT_SEARCH_MAX_FIELD_CHARS,
+  DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+  DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD,
   DEFAULT_SEARCH_MIN_TOKEN_LENGTH,
+  isSearchFieldBlocklisted,
   resolveSearchConfig,
   resolveSearchMinTokenLength,
+  resolveSearchTokenLimits,
 } from '../config'
 
 describe('resolveSearchMinTokenLength', () => {
@@ -40,3 +45,175 @@ describe('resolveSearchMinTokenLength', () => {
     expect(resolveSearchConfig().minTokenLength).toBe(resolveSearchMinTokenLength())
   })
 })
+
+describe('OM_SEARCH_FIELD_BLOCKLIST parsing', () => {
+  const originalValue = process.env.OM_SEARCH_FIELD_BLOCKLIST
+
+  afterEach(() => {
+    if (originalValue === undefined) {
+      delete process.env.OM_SEARCH_FIELD_BLOCKLIST
+    } else {
+      process.env.OM_SEARCH_FIELD_BLOCKLIST = originalValue
+    }
+  })
+
+  it('always includes the built-in defaults', () => {
+    delete process.env.OM_SEARCH_FIELD_BLOCKLIST
+    const config = resolveSearchConfig()
+    expect(config.blocklistedFields).toEqual(expect.arrayContaining(['password', 'token', 'secret', 'hash']))
+    expect(config.entityBlocklistedFields).toEqual({})
+  })
+
+  it('treats an unprefixed entry as global and lowercases it', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = ' Body , body '
+    const config = resolveSearchConfig()
+    expect(config.blocklistedFields.filter((entry) => entry === 'body')).toHaveLength(1)
+    expect(config.entityBlocklistedFields).toEqual({})
+  })
+
+  it('routes an entityType@field entry into the per-entity map only', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = 'customers:customer_interaction@body'
+    const config = resolveSearchConfig()
+    expect(config.blocklistedFields).not.toContain('body')
+    expect(config.entityBlocklistedFields).toEqual({ 'customers:customer_interaction': ['body'] })
+  })
+
+  it('groups several fields under the same entity type', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = 'customers:customer_interaction@body,customers:customer_interaction@subject'
+    expect(resolveSearchConfig().entityBlocklistedFields).toEqual({
+      'customers:customer_interaction': ['body', 'subject'],
+    })
+  })
+
+  it('ignores entries whose field part is empty', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = 'customers:customer_interaction@,@,body'
+    const config = resolveSearchConfig()
+    expect(config.blocklistedFields).toContain('body')
+    expect(config.entityBlocklistedFields).toEqual({})
+  })
+
+  it('treats a leading separator as a global entry', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = '@body'
+    const config = resolveSearchConfig()
+    expect(config.blocklistedFields).toContain('body')
+    expect(config.entityBlocklistedFields).toEqual({})
+  })
+
+  it('stores entity types that collide with inherited object keys as own entries', () => {
+    process.env.OM_SEARCH_FIELD_BLOCKLIST = 'constructor@body,toString@summary,__proto__@notes'
+    const config = resolveSearchConfig()
+
+    expect(Object.getPrototypeOf(config.entityBlocklistedFields)).toBeNull()
+    expect(config.entityBlocklistedFields?.['constructor']).toEqual(['body'])
+    expect(config.entityBlocklistedFields?.['tostring']).toEqual(['summary'])
+    expect(config.entityBlocklistedFields?.['__proto__']).toEqual(['notes'])
+    expect(isSearchFieldBlocklisted('body', 'constructor', config)).toBe(true)
+    expect(isSearchFieldBlocklisted('summary', 'toString', config)).toBe(true)
+    expect(isSearchFieldBlocklisted('notes', '__proto__', config)).toBe(true)
+  })
+})
+
+describe('search token limits', () => {
+  const variableNames = [
+    'OM_SEARCH_MAX_FIELD_CHARS',
+    'OM_SEARCH_MAX_TOKENS_PER_FIELD',
+    'OM_SEARCH_MAX_TOKENS_PER_RECORD',
+  ] as const
+  const originalValues = Object.fromEntries(variableNames.map((name) => [name, process.env[name]]))
+
+  afterEach(() => {
+    for (const name of variableNames) {
+      const original = originalValues[name]
+      if (original === undefined) delete process.env[name]
+      else process.env[name] = original
+    }
+  })
+
+  it('uses safe defaults when limits are unset', () => {
+    for (const name of variableNames) delete process.env[name]
+
+    expect(resolveSearchTokenLimits(resolveSearchConfig())).toEqual({
+      maxFieldChars: DEFAULT_SEARCH_MAX_FIELD_CHARS,
+      maxTokensPerField: DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+      maxTokensPerRecord: DEFAULT_SEARCH_MAX_TOKENS_PER_RECORD,
+    })
+  })
+
+  it('accepts zero to disable individual limits', () => {
+    for (const name of variableNames) process.env[name] = '0'
+
+    expect(resolveSearchTokenLimits(resolveSearchConfig())).toEqual({
+      maxFieldChars: 0,
+      maxTokensPerField: 0,
+      maxTokensPerRecord: 0,
+    })
+  })
+
+  it('normalizes invalid custom config values to defaults', () => {
+    expect(resolveSearchTokenLimits({
+      enabled: true,
+      minTokenLength: 3,
+      enablePartials: true,
+      hashAlgorithm: 'sha256',
+      storeRawTokens: false,
+      blocklistedFields: [],
+      maxFieldChars: Number.NaN,
+      maxTokensPerField: -1,
+      maxTokensPerRecord: 4.8,
+    })).toEqual({
+      maxFieldChars: DEFAULT_SEARCH_MAX_FIELD_CHARS,
+      maxTokensPerField: DEFAULT_SEARCH_MAX_TOKENS_PER_FIELD,
+      maxTokensPerRecord: 4,
+    })
+  })
+})
+
+describe('isSearchFieldBlocklisted', () => {
+  const baseConfig = {
+    enabled: true,
+    minTokenLength: 3,
+    enablePartials: true,
+    hashAlgorithm: 'sha256' as const,
+    storeRawTokens: false,
+  }
+
+  it('matches global entries by substring for any entity type', () => {
+    const config = { ...baseConfig, blocklistedFields: ['password'], entityBlocklistedFields: {} }
+    expect(isSearchFieldBlocklisted('password_hash', 'customers:person', config)).toBe(true)
+    expect(isSearchFieldBlocklisted('password_hash', null, config)).toBe(true)
+    expect(isSearchFieldBlocklisted('display_name', 'customers:person', config)).toBe(false)
+  })
+
+  it('applies an entity-scoped entry only to its own entity type', () => {
+    const config = {
+      ...baseConfig,
+      blocklistedFields: [],
+      entityBlocklistedFields: { 'customers:customer_interaction': ['body'] },
+    }
+    expect(isSearchFieldBlocklisted('body', 'customers:customer_interaction', config)).toBe(true)
+    expect(isSearchFieldBlocklisted('body', 'customers:person', config)).toBe(false)
+    expect(isSearchFieldBlocklisted('body', null, config)).toBe(false)
+  })
+
+  it('compares entity types case-insensitively', () => {
+    const config = {
+      ...baseConfig,
+      blocklistedFields: [],
+      entityBlocklistedFields: { 'customers:customer_interaction': ['body'] },
+    }
+    expect(isSearchFieldBlocklisted('BODY', ' Customers:Customer_Interaction ', config)).toBe(true)
+  })
+
+  it('ignores inherited Object.prototype keys when looking up an entity type', () => {
+    const config = { ...baseConfig, blocklistedFields: [], entityBlocklistedFields: {} }
+    expect(isSearchFieldBlocklisted('body', 'constructor', config)).toBe(false)
+    expect(isSearchFieldBlocklisted('body', 'toString', config)).toBe(false)
+    expect(isSearchFieldBlocklisted('body', '__proto__', config)).toBe(false)
+  })
+
+  it('tolerates a config without the per-entity map', () => {
+    const config = { ...baseConfig, blocklistedFields: ['secret'] }
+    expect(isSearchFieldBlocklisted('client_secret', 'customers:person', config)).toBe(true)
+    expect(isSearchFieldBlocklisted('body', 'customers:person', config)).toBe(false)
+  })
+})

package/src/lib/search/__tests__/tokenLookup.test.ts

@@ -0,0 +1,206 @@
+import {
+  findEntityIdsBySearchTokens,
+  findEntityIdsBySearchTokensCompat,
+} from '../tokenLookup'
+import { resolveSearchConfig } from '../config'
+import { tokenizeText } from '../tokenize'
+
+type KyselyCall = {
+  method: string
+  args: unknown[]
+}
+
+function createKyselyMock(rows: Array<{ entity_id: unknown }>) {
+  const calls: KyselyCall[] = []
+  const tableNameRef: { value: string | null } = { value: null }
+  const builder: Record<string, unknown> = {}
+  const passthrough = (method: string) =>
+    (...args: unknown[]) => {
+      calls.push({ method, args })
+      return builder
+    }
+  builder.select = passthrough('select')
+  builder.where = passthrough('where')
+  builder.groupBy = passthrough('groupBy')
+  builder.having = passthrough('having')
+  builder.execute = jest.fn(async () => rows)
+
+  const db = {
+    selectFrom: (table: string) => {
+      tableNameRef.value = table
+      return builder
+    },
+  }
+  return { db: db as never, calls, tableNameRef, builder }
+}
+
+function compileSql(raw: unknown): string {
+  if (raw && typeof raw === 'object' && 'toOperationNode' in raw && typeof raw.toOperationNode === 'function') {
+    const node = raw.toOperationNode() as { sqlFragments?: unknown }
+    return Array.isArray(node.sqlFragments) ? node.sqlFragments.join(' ? ') : ''
+  }
+  if (raw && typeof raw === 'object' && 'sql' in raw) return String(raw.sql)
+  return String(raw)
+}
+
+function rawWhereFragments(calls: KyselyCall[]): string[] {
+  return calls
+    .filter((call) => call.method === 'where' && call.args.length === 1)
+    .map((call) => compileSql(call.args[0]))
+}
+
+describe('findEntityIdsBySearchTokens', () => {
+  const baseInput = {
+    entityType: 'customers:customer_entity',
+    query: 'Hello',
+  }
+
+  it('skips the lookup for a blank query', async () => {
+    const { db, builder } = createKyselyMock([])
+    const result = await findEntityIdsBySearchTokens({ ...baseInput, db, query: '   ' })
+    expect(result).toEqual({ matched: false, reason: 'empty-query' })
+    expect(builder.execute).not.toHaveBeenCalled()
+  })
+
+  it('skips the lookup when token search is disabled', async () => {
+    const { db, builder } = createKyselyMock([])
+    const result = await findEntityIdsBySearchTokens({
+      ...baseInput,
+      db,
+      config: { ...resolveSearchConfig(), enabled: false },
+    })
+    expect(result).toEqual({ matched: false, reason: 'search-disabled' })
+    expect(builder.execute).not.toHaveBeenCalled()
+  })
+
+  it('skips the lookup when the query yields no indexable tokens', async () => {
+    const { db, builder } = createKyselyMock([])
+    const result = await findEntityIdsBySearchTokens({ ...baseInput, db, query: '!' })
+    expect(result).toEqual({ matched: false, reason: 'no-tokens' })
+    expect(builder.execute).not.toHaveBeenCalled()
+  })
+
+  it('requires every query token to be present on the record', async () => {
+    const { db, calls } = createKyselyMock([{ entity_id: 'a' }])
+    await findEntityIdsBySearchTokens({ ...baseInput, db, query: 'ada lovelace' })
+
+    const expectedHashes = tokenizeText('ada lovelace', resolveSearchConfig()).hashes
+    const hashFilter = calls.find((call) => call.args[0] === 'token_hash' && call.args[1] === 'in')
+    expect(hashFilter?.args[2]).toEqual(expectedHashes)
+
+    const having = calls.find((call) => call.method === 'having')
+    expect(compileSql(having?.args[0])).toContain('count(distinct token_hash) >=')
+  })
+
+  it('returns the matched ids and drops non-string rows', async () => {
+    const { db, tableNameRef } = createKyselyMock([
+      { entity_id: 'id-1' },
+      { entity_id: null },
+      { entity_id: 42 },
+      { entity_id: '' },
+      { entity_id: 'id-2' },
+    ])
+    const result = await findEntityIdsBySearchTokens({ ...baseInput, db })
+    expect(result).toEqual({ matched: true, ids: ['id-1', 'id-2'] })
+    expect(tableNameRef.value).toBe('search_tokens')
+  })
+
+  it('reports an empty match instead of a skip when nothing matched', async () => {
+    const { db } = createKyselyMock([])
+    const result = await findEntityIdsBySearchTokens({ ...baseInput, db })
+    expect(result).toEqual({ matched: true, ids: [] })
+  })
+
+  it('narrows a single field with equality and multiple fields with IN', async () => {
+    const single = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db: single.db, fields: ['name'] })
+    expect(single.calls).toContainEqual({ method: 'where', args: ['field', '=', 'name'] })
+
+    const multi = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db: multi.db, fields: ['name', 'email'] })
+    expect(multi.calls).toContainEqual({ method: 'where', args: ['field', 'in', ['name', 'email']] })
+  })
+
+  it('omits the field predicate when no fields are supplied', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db, fields: [] })
+    expect(calls.some((call) => call.args[0] === 'field')).toBe(false)
+  })
+
+  it('omits tenant and organization predicates when the scope is absent', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db })
+    expect(rawWhereFragments(calls)).toEqual([])
+    expect(calls.some((call) => call.args[0] === 'organization_id')).toBe(false)
+  })
+
+  it('emits a null-safe tenant predicate for an explicit null tenant', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db, scope: { tenantId: null } })
+    expect(rawWhereFragments(calls).some((s) => s.includes('tenant_id is not distinct from'))).toBe(true)
+  })
+
+  it('emits a null-safe organization predicate for an explicit null organization', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({ ...baseInput, db, scope: { organizationId: null } })
+    expect(rawWhereFragments(calls).some((s) => s.includes('organization_id is not distinct from'))).toBe(true)
+  })
+
+  it('prefers a concrete organization over the visible-organization list', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({
+      ...baseInput,
+      db,
+      scope: { organizationId: 'org-1', organizationIds: ['org-1', 'org-2'] },
+    })
+    expect(calls).toContainEqual({ method: 'where', args: ['organization_id', '=', 'org-1'] })
+    expect(calls.some((call) => call.args[1] === 'in' && call.args[0] === 'organization_id')).toBe(false)
+  })
+
+  it('falls back to the visible-organization list when no organization is selected', async () => {
+    const { db, calls } = createKyselyMock([])
+    await findEntityIdsBySearchTokens({
+      ...baseInput,
+      db,
+      scope: { organizationIds: ['org-1', 'org-2'] },
+    })
+    expect(calls).toContainEqual({ method: 'where', args: ['organization_id', 'in', ['org-1', 'org-2']] })
+  })
+})
+
+describe('findEntityIdsBySearchTokensCompat', () => {
+  it('returns null only for a blank query', async () => {
+    const { db } = createKyselyMock([])
+    await expect(findEntityIdsBySearchTokensCompat({
+      db,
+      entityType: 'customers:customer_entity',
+      query: ' ',
+    })).resolves.toBeNull()
+  })
+
+  it('returns an empty array for the remaining skip reasons', async () => {
+    const noTokens = createKyselyMock([])
+    await expect(findEntityIdsBySearchTokensCompat({
+      db: noTokens.db,
+      entityType: 'customers:customer_entity',
+      query: '!',
+    })).resolves.toEqual([])
+
+    const disabled = createKyselyMock([])
+    await expect(findEntityIdsBySearchTokensCompat({
+      db: disabled.db,
+      entityType: 'customers:customer_entity',
+      query: 'Hello',
+      config: { ...resolveSearchConfig(), enabled: false },
+    })).resolves.toEqual([])
+  })
+
+  it('returns the matched ids', async () => {
+    const { db } = createKyselyMock([{ entity_id: 'id-1' }])
+    await expect(findEntityIdsBySearchTokensCompat({
+      db,
+      entityType: 'customers:customer_entity',
+      query: 'Hello',
+    })).resolves.toEqual(['id-1'])
+  })
+})

package/src/lib/search/__tests__/tokenize.test.ts

@@ -0,0 +1,49 @@
+import type { SearchConfig } from '../config'
+import { tokenizeText } from '../tokenize'
+
+const baseConfig: SearchConfig = {
+  enabled: true,
+  minTokenLength: 3,
+  enablePartials: true,
+  hashAlgorithm: 'sha256',
+  storeRawTokens: false,
+  blocklistedFields: [],
+}
+
+describe('tokenizeText limits', () => {
+  test('truncates oversized field text before tokenizing', () => {
+    const config = { ...baseConfig, enablePartials: false, maxFieldChars: 10 }
+
+    const { tokens } = tokenizeText('aaaa bbbb cccc', config)
+
+    expect(tokens).toEqual(['aaaa', 'bbbb'])
+  })
+
+  test('bounds prefix expansion while collecting tokens', () => {
+    const config = { ...baseConfig, maxFieldChars: 100, maxTokensPerField: 5 }
+
+    const { tokens, hashes } = tokenizeText('a'.repeat(100), config)
+
+    expect(tokens).toEqual(['aaa', 'aaaa', 'aaaaa', 'aaaaaa', 'aaaaaaa'])
+    expect(hashes).toHaveLength(5)
+  })
+
+  test('applies safe defaults to legacy configs without limit fields', () => {
+    const { tokens } = tokenizeText('a'.repeat(50_000), baseConfig)
+
+    expect(tokens).toHaveLength(5_000)
+  })
+
+  test('allows a limit to be disabled explicitly', () => {
+    const config = {
+      ...baseConfig,
+      enablePartials: false,
+      maxFieldChars: 0,
+      maxTokensPerField: 0,
+    }
+
+    const { tokens } = tokenizeText('alpha beta gamma delta', config)
+
+    expect(tokens).toEqual(['alpha', 'beta', 'gamma', 'delta'])
+  })
+})

package/src/lib/search/auto-indexing.ts

@@ -0,0 +1,22 @@
+import { parseBooleanToken } from '../boolean'
+
+/**
+ * Module id backing the tenant-scoped vector auto-indexing switch. Kept as `vector`
+ * (rather than `search`) for backwards compatibility with rows written before the
+ * search module absorbed the vector settings.
+ */
+export const SEARCH_AUTO_INDEX_CONFIG_MODULE = 'vector'
+export const SEARCH_AUTO_INDEX_CONFIG_KEY = 'auto_index_enabled'
+
+/**
+ * Instance-wide kill switch for vector auto-indexing. Lives in `@open-mercato/shared`
+ * so consumers outside `@open-mercato/search` (for example the query index status page)
+ * can report vector coverage honestly without duplicating the env-var contract.
+ */
+export function envDisablesAutoIndexing(): boolean {
+  const raw =
+    process.env.OM_DISABLE_VECTOR_SEARCH_AUTOINDEXING ??
+    process.env.DISABLE_VECTOR_SEARCH_AUTOINDEXING
+  if (!raw) return false
+  return parseBooleanToken(raw) === true
+}