@open-mercato/search 0.6.8-develop.7100.1.fbf66fca35 → 0.7.0

package/src/service.ts

@@ -25,38 +25,6 @@ const DEFAULT_MERGE_CONFIG: ResultMergeConfig = {
  */
 const STRATEGY_AVAILABILITY_CACHE_TTL_MS = 2_000
 
-/**
- * Maximum records indexed at once when bulkIndex falls back to per-record writes
- * for a strategy that has no bulkIndex implementation (currently the vector
- * strategy, whose index() performs an embedding-provider round trip per record).
- * A whole reindex page arrives in one bulkIndex call, so an unbounded fan-out
- * would burst hundreds of concurrent provider requests from a single job.
- */
-const BULK_INDEX_FALLBACK_CONCURRENCY = 4
-
-/**
- * Map items through an async worker with a fixed number of in-flight calls.
- * Rejects with the first error, matching Promise.all semantics.
- */
-async function mapWithConcurrency<T>(
-  items: T[],
-  limit: number,
-  worker: (item: T) => Promise<void>,
-): Promise<void> {
-  if (items.length === 0) return
-
-  let nextIndex = 0
-  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
-    for (;;) {
-      const currentIndex = nextIndex++
-      if (currentIndex >= items.length) return
-      await worker(items[currentIndex])
-    }
-  })
-
-  await Promise.all(runners)
-}
-
 function normalizeOrganizationFilter(options: SearchOptions): string[] | null {
   const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''
   if (single) return [single]
@@ -267,11 +235,8 @@ export class SearchService {
         if (strategy.bulkIndex) {
           return strategy.bulkIndex(records)
         }
-        // Fallback to individual indexing, bounded so a strategy without a batch
-        // implementation cannot turn one batch job into hundreds of concurrent writes.
-        return mapWithConcurrency(records, BULK_INDEX_FALLBACK_CONCURRENCY, (record) =>
-          this.executeStrategyIndex(strategy, record),
-        )
+        // Fallback to individual indexing
+        return Promise.all(records.map((record) => this.executeStrategyIndex(strategy, record)))
       }),
     )
 

package/src/strategies/token.strategy.ts

@@ -67,21 +67,10 @@ export class TokenSearchStrategy implements SearchStrategy {
     // Dynamically import tokenization to avoid circular dependencies
     const { tokenizeText } = await import('@open-mercato/shared/lib/search/tokenize')
     const { resolveSearchConfig } = await import('@open-mercato/shared/lib/search/config')
-    const { listSearchTokenExcludedEntityTypes } = await import(
-      '@open-mercato/core/modules/query_index/lib/search-entity-policy'
-    )
 
     const config = resolveSearchConfig()
     if (!config.enabled) return []
 
-    // The rows themselves stay in `search_tokens` — list routes and the query engines' encrypted
-    // like/ilike rewrite depend on them — so the exclusion is enforced here, at read time.
-    const excludedEntityTypes = listSearchTokenExcludedEntityTypes()
-    const requestedEntityTypes = options.entityTypes?.length
-      ? options.entityTypes.filter((entityType) => !excludedEntityTypes.includes(entityType))
-      : undefined
-    if (options.entityTypes?.length && !requestedEntityTypes?.length) return []
-
     const { hashes } = tokenizeText(query, config)
     if (hashes.length === 0) return []
 
@@ -107,10 +96,8 @@ export class TokenSearchStrategy implements SearchStrategy {
       queryBuilder = queryBuilder.where('organization_id' as any, 'in', organizationIds)
     }
 
-    if (requestedEntityTypes?.length) {
-      queryBuilder = queryBuilder.where('entity_type' as any, 'in', requestedEntityTypes)
-    } else if (excludedEntityTypes.length) {
-      queryBuilder = queryBuilder.where('entity_type' as any, 'not in', excludedEntityTypes)
+    if (options.entityTypes?.length) {
+      queryBuilder = queryBuilder.where('entity_type' as any, 'in', options.entityTypes)
     }
 
     const rows = await queryBuilder.execute() as Array<{

package/src/__tests__/search-indexer-batch.test.ts

@@ -1,214 +0,0 @@
-import { SearchIndexer } from '../indexer/search-indexer'
-import type { SearchModuleConfig } from '../types'
-import type { QueryEngine, QueryResult } from '@open-mercato/shared/lib/query/types'
-
-describe('SearchIndexer.indexRecordsById', () => {
-  const moduleConfigs: SearchModuleConfig[] = [
-    {
-      entities: [
-        {
-          entityId: 'test:entity',
-          enabled: true,
-          formatResult: async (ctx) => ({ title: String(ctx.record.name ?? ctx.record.id) }),
-        },
-        {
-          entityId: 'test:other',
-          enabled: true,
-          formatResult: async (ctx) => ({ title: String(ctx.record.name ?? ctx.record.id) }),
-        },
-      ],
-    },
-  ]
-
-  function makeQueryEngine(recordsByEntity: Record<string, Record<string, unknown>[]>): QueryEngine {
-    return {
-      query: jest.fn(async (entity, opts) => {
-        const wantedId = (opts?.filters as { id?: string } | undefined)?.id
-        const items = (recordsByEntity[entity as string] ?? []).filter((r) => r.id === wantedId)
-        return { items, total: items.length } as QueryResult
-      }),
-    }
-  }
-
-  it('writes N queued records through exactly one bulkIndex call, not N', async () => {
-    const records = [
-      { id: 'rec-1', name: 'Alpha' },
-      { id: 'rec-2', name: 'Beta' },
-      { id: 'rec-3', name: 'Gamma' },
-    ]
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
-      queryEngine: makeQueryEngine({ 'test:entity': records }),
-    })
-
-    const result = await indexer.indexRecordsById({
-      items: [
-        { entityId: 'test:entity', recordId: 'rec-1' },
-        { entityId: 'test:entity', recordId: 'rec-2' },
-        { entityId: 'test:entity', recordId: 'rec-3' },
-      ],
-      tenantId: 'tenant-123',
-      organizationId: 'org-456',
-    })
-
-    expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
-    expect(searchService.bulkIndex).toHaveBeenCalledWith([
-      expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-1', tenantId: 'tenant-123', organizationId: 'org-456' }),
-      expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-2', tenantId: 'tenant-123', organizationId: 'org-456' }),
-      expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-3', tenantId: 'tenant-123', organizationId: 'org-456' }),
-    ])
-    expect(result).toEqual({ indexed: 3, skipped: 0 })
-  })
-
-  it('collapses records spanning multiple entities into one bulkIndex call', async () => {
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
-      queryEngine: makeQueryEngine({
-        'test:entity': [{ id: 'rec-1', name: 'Alpha' }],
-        'test:other': [{ id: 'other-1', name: 'Delta' }],
-      }),
-    })
-
-    const result = await indexer.indexRecordsById({
-      items: [
-        { entityId: 'test:entity', recordId: 'rec-1' },
-        { entityId: 'test:other', recordId: 'other-1' },
-      ],
-      tenantId: 'tenant-123',
-      organizationId: null,
-    })
-
-    expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
-    expect(searchService.bulkIndex).toHaveBeenCalledWith([
-      expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-1' }),
-      expect.objectContaining({ entityId: 'test:other', recordId: 'other-1' }),
-    ])
-    expect(result).toEqual({ indexed: 2, skipped: 0 })
-  })
-
-  it('loads each record with custom fields and without triggering auto-reindex', async () => {
-    const queryEngine = makeQueryEngine({ 'test:entity': [{ id: 'rec-1', name: 'Alpha' }] })
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
-
-    await indexer.indexRecordsById({
-      items: [{ entityId: 'test:entity', recordId: 'rec-1' }],
-      tenantId: 'tenant-123',
-      organizationId: 'org-456',
-    })
-
-    expect(queryEngine.query).toHaveBeenCalledWith(
-      'test:entity',
-      expect.objectContaining({
-        tenantId: 'tenant-123',
-        organizationId: 'org-456',
-        filters: { id: 'rec-1' },
-        includeCustomFields: true,
-        skipAutoReindex: true,
-      }),
-    )
-  })
-
-  it('skips records that no longer exist without failing the batch write', async () => {
-    const records = [{ id: 'rec-1', name: 'Alpha' }]
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
-      queryEngine: makeQueryEngine({ 'test:entity': records }),
-    })
-
-    const result = await indexer.indexRecordsById({
-      items: [
-        { entityId: 'test:entity', recordId: 'rec-1' },
-        { entityId: 'test:entity', recordId: 'missing' },
-      ],
-      tenantId: 'tenant-123',
-      organizationId: null,
-    })
-
-    expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
-    expect(result).toEqual({ indexed: 1, skipped: 1 })
-  })
-
-  it('keeps indexing the batch when loading one record throws', async () => {
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const queryEngine: QueryEngine = {
-      query: jest.fn(async (_entity, opts) => {
-        const wantedId = (opts?.filters as { id?: string } | undefined)?.id
-        if (wantedId === 'boom') throw new Error('connection lost')
-        return { items: [{ id: wantedId, name: 'Alpha' }], total: 1 } as QueryResult
-      }),
-    }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
-
-    const result = await indexer.indexRecordsById({
-      items: [
-        { entityId: 'test:entity', recordId: 'rec-1' },
-        { entityId: 'test:entity', recordId: 'boom' },
-        { entityId: 'test:entity', recordId: 'rec-2' },
-      ],
-      tenantId: 'tenant-123',
-      organizationId: null,
-    })
-
-    expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
-    expect(searchService.bulkIndex).toHaveBeenCalledWith([
-      expect.objectContaining({ recordId: 'rec-1' }),
-      expect.objectContaining({ recordId: 'rec-2' }),
-    ])
-    expect(result).toEqual({ indexed: 2, skipped: 1 })
-  })
-
-  it('propagates a bulkIndex failure so the queue can retry the job', async () => {
-    const searchService = { bulkIndex: jest.fn().mockRejectedValue(new Error('meilisearch unavailable')) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
-      queryEngine: makeQueryEngine({ 'test:entity': [{ id: 'rec-1', name: 'Alpha' }] }),
-    })
-
-    await expect(
-      indexer.indexRecordsById({
-        items: [{ entityId: 'test:entity', recordId: 'rec-1' }],
-        tenantId: 'tenant-123',
-        organizationId: null,
-      }),
-    ).rejects.toThrow('meilisearch unavailable')
-  })
-
-  it('skips entities that are not configured and never calls bulkIndex when nothing is indexable', async () => {
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
-      queryEngine: makeQueryEngine({}),
-    })
-
-    const result = await indexer.indexRecordsById({
-      items: [{ entityId: 'unknown:entity', recordId: 'rec-1' }],
-      tenantId: 'tenant-123',
-      organizationId: null,
-    })
-
-    expect(searchService.bulkIndex).not.toHaveBeenCalled()
-    expect(result).toEqual({ indexed: 0, skipped: 1 })
-  })
-
-  it('counts records dropped for a missing id as skipped so the totals add up', async () => {
-    const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
-    const queryEngine: QueryEngine = {
-      query: jest.fn(async (_entity, opts) => {
-        const wantedId = (opts?.filters as { id?: string } | undefined)?.id
-        const item = wantedId === 'no-id' ? { name: 'Ghost' } : { id: wantedId, name: 'Alpha' }
-        return { items: [item], total: 1 } as QueryResult
-      }),
-    }
-    const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
-
-    const result = await indexer.indexRecordsById({
-      items: [
-        { entityId: 'test:entity', recordId: 'rec-1' },
-        { entityId: 'test:entity', recordId: 'no-id' },
-      ],
-      tenantId: 'tenant-123',
-      organizationId: null,
-    })
-
-    expect(result).toEqual({ indexed: 1, skipped: 1 })
-  })
-})

package/src/__tests__/token-strategy-entity-exclusion.test.ts

@@ -1,99 +0,0 @@
-import { TokenSearchStrategy } from '../strategies/token.strategy'
-
-/**
- * Coverage for `OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY` on the read path (#5046).
- *
- * The exclusion is read-side by design and the writer deliberately ignores the flag:
- * `search_tokens` doubles as the encrypted-column lookup index, so the People and Companies list
- * search resolves ids through the very `customers:customer_entity` rows this flag hides from
- * search results. Dropping them write-side would turn that list search into a silent empty page.
- * These tests therefore pin the SQL predicate the strategy issues, never the writer's behavior —
- * `packages/core/src/modules/query_index/__tests__/search-entity-policy.test.ts` is the other half
- * of the pair and asserts the writer emits byte-identical rows in both flag states.
- */
-
-const CUSTOMER_ENTITY = 'customers:customer_entity'
-const PERSON_PROFILE = 'customers:customer_person_profile'
-
-type RecordedWhere = [string, string, unknown]
-
-function createMockDb() {
-  const wheres: RecordedWhere[] = []
-  const builder: Record<string, unknown> = {
-    select: jest.fn(() => builder),
-    where: jest.fn((column: unknown, op?: unknown, value?: unknown) => {
-      if (typeof column === 'string' && typeof op === 'string') wheres.push([column, op, value])
-      return builder
-    }),
-    groupBy: jest.fn(() => builder),
-    having: jest.fn(() => builder),
-    orderBy: jest.fn(() => builder),
-    limit: jest.fn(() => builder),
-    execute: jest.fn().mockResolvedValue([]),
-  }
-  const db = { selectFrom: jest.fn(() => builder) }
-  return { db, wheres, builder }
-}
-
-const entityTypePredicates = (wheres: RecordedWhere[]) => wheres.filter(([column]) => column === 'entity_type')
-
-const originalFlag = process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY
-
-afterEach(() => {
-  if (originalFlag === undefined) delete process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY
-  else process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY = originalFlag
-})
-
-describe('TokenSearchStrategy excludes base customer entities by default', () => {
-  it('adds a NOT IN predicate when the caller requests no specific entity types', async () => {
-    delete process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY
-    const { db, wheres } = createMockDb()
-    const strategy = new TokenSearchStrategy(db as never)
-
-    await strategy.search('ada lovelace', { tenantId: 'tenant-1' })
-
-    expect(entityTypePredicates(wheres)).toEqual([['entity_type', 'not in', [CUSTOMER_ENTITY]]])
-  })
-
-  it('drops the excluded type from an explicit entityTypes filter and keeps the rest', async () => {
-    delete process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY
-    const { db, wheres } = createMockDb()
-    const strategy = new TokenSearchStrategy(db as never)
-
-    await strategy.search('ada lovelace', {
-      tenantId: 'tenant-1',
-      entityTypes: [CUSTOMER_ENTITY, PERSON_PROFILE],
-    })
-
-    expect(entityTypePredicates(wheres)).toEqual([['entity_type', 'in', [PERSON_PROFILE]]])
-  })
-
-  it('returns no results — and issues no query — when only excluded types were requested', async () => {
-    delete process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY
-    const { db } = createMockDb()
-    const strategy = new TokenSearchStrategy(db as never)
-
-    const results = await strategy.search('ada lovelace', {
-      tenantId: 'tenant-1',
-      entityTypes: [CUSTOMER_ENTITY],
-    })
-
-    expect(results).toEqual([])
-    expect(db.selectFrom).not.toHaveBeenCalled()
-  })
-
-  it('leaves the query untouched when the flag re-enables base customer entities', async () => {
-    process.env.OM_SEARCH_CUSTOMERS_INDEX_BASE_ENTITY = 'true'
-    const { db, wheres } = createMockDb()
-    const strategy = new TokenSearchStrategy(db as never)
-
-    await strategy.search('ada lovelace', { tenantId: 'tenant-1' })
-    expect(entityTypePredicates(wheres)).toEqual([])
-
-    await strategy.search('ada lovelace', {
-      tenantId: 'tenant-1',
-      entityTypes: [CUSTOMER_ENTITY, PERSON_PROFILE],
-    })
-    expect(entityTypePredicates(wheres)).toEqual([['entity_type', 'in', [CUSTOMER_ENTITY, PERSON_PROFILE]]])
-  })
-})