@open-mercato/search 0.6.8-develop.7096.1.97319f09f6 → 0.6.8-develop.7100.1.fbf66fca35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/indexer/search-indexer.js +66 -0
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/modules/search/workers/fulltext-index.worker.js +7 -24
- package/dist/modules/search/workers/fulltext-index.worker.js.map +2 -2
- package/dist/service.js +18 -1
- package/dist/service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/search-indexer-batch.test.ts +214 -0
- package/src/__tests__/service.test.ts +24 -0
- package/src/__tests__/workers.test.ts +46 -17
- package/src/indexer/search-indexer.ts +84 -0
- package/src/modules/search/workers/fulltext-index.worker.ts +13 -29
- package/src/service.ts +37 -2
package/dist/service.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/service.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n SearchStrategy,\n SearchStrategyId,\n SearchOptions,\n SearchResult,\n SearchServiceOptions,\n ResultMergeConfig,\n IndexableRecord,\n PresenterEnricherFn,\n} from './types'\nimport { mergeAndRankResults } from './lib/merger'\nimport { searchError } from './lib/debug'\n\n/**\n * Default merge configuration.\n */\nconst DEFAULT_MERGE_CONFIG: ResultMergeConfig = {\n duplicateHandling: 'highest_score',\n}\n\n/**\n * Cache TTL for strategy availability checks.\n * Short window so connectivity changes (Meilisearch up/down) propagate quickly,\n * long enough to skip per-request RTT to remote backends on hot paths.\n */\nconst STRATEGY_AVAILABILITY_CACHE_TTL_MS = 2_000\n\nfunction normalizeOrganizationFilter(options: SearchOptions): string[] | null {\n const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''\n if (single) return [single]\n if (!Array.isArray(options.organizationIds)) return null\n\n const values = Array.from(new Set(\n options.organizationIds\n .map((value) => (typeof value === 'string' ? value.trim() : ''))\n .filter((value) => value.length > 0),\n ))\n return values\n}\n\nfunction filterResultsByOrganizationScope(results: SearchResult[], options: SearchOptions): SearchResult[] {\n const organizationIds = normalizeOrganizationFilter(options)\n if (!organizationIds) return results\n if (organizationIds.length === 0) return []\n\n const allowed = new Set(organizationIds)\n return results.filter((result) => {\n const organizationId = typeof result.organizationId === 'string' ? result.organizationId.trim() : ''\n return organizationId.length > 0 && allowed.has(organizationId)\n })\n}\n\n/**\n * SearchService orchestrates multiple search strategies, executing searches in parallel\n * and merging results using the RRF algorithm.\n *\n * Features:\n * - Parallel strategy execution for optimal performance\n * - Graceful degradation when strategies fail\n * - Result merging with configurable weights\n * - Strategy availability checking\n *\n * @example\n * ```typescript\n * const service = new SearchService({\n * strategies: [tokenStrategy, vectorStrategy, fulltextStrategy],\n * defaultStrategies: ['fulltext', 'vector', 'tokens'],\n * mergeConfig: {\n * duplicateHandling: 'highest_score',\n * strategyWeights: { fulltext: 1.2, vector: 1.0, tokens: 0.8 },\n * },\n * })\n *\n * const results = await service.search('john doe', { tenantId: 'tenant-123' })\n * ```\n */\nexport class SearchService {\n private readonly strategies: Map<SearchStrategyId, SearchStrategy>\n private readonly defaultStrategies: SearchStrategyId[]\n private readonly fallbackStrategy: SearchStrategyId | undefined\n private readonly mergeConfig: ResultMergeConfig\n private readonly presenterEnricher?: PresenterEnricherFn\n private readonly availabilityCache = new Map<SearchStrategyId, { value: boolean; expiresAt: number }>()\n private readonly availabilityInflight = new Map<SearchStrategyId, Promise<boolean>>()\n private readonly availabilityCacheTtlMs: number\n\n constructor(options: SearchServiceOptions = {}) {\n this.strategies = new Map()\n for (const strategy of options.strategies ?? []) {\n this.strategies.set(strategy.id, strategy)\n }\n this.defaultStrategies = options.defaultStrategies ?? ['tokens']\n this.fallbackStrategy = options.fallbackStrategy\n this.mergeConfig = options.mergeConfig ?? DEFAULT_MERGE_CONFIG\n this.presenterEnricher = options.presenterEnricher\n this.availabilityCacheTtlMs = options.availabilityCacheTtlMs ?? STRATEGY_AVAILABILITY_CACHE_TTL_MS\n }\n\n /**\n * Get all registered strategies.\n */\n getStrategies(): SearchStrategy[] {\n return Array.from(this.strategies.values())\n }\n\n /**\n * Execute a search query across configured strategies.\n *\n * @param query - Search query string\n * @param options - Search options with tenant, filters, etc.\n * @returns Merged and ranked search results\n */\n async search(query: string, options: SearchOptions): Promise<SearchResult[]> {\n const organizationIds = normalizeOrganizationFilter(options)\n if (organizationIds && organizationIds.length === 0) {\n return []\n }\n\n const strategyIds = options.strategies ?? this.defaultStrategies\n const activeStrategies = await this.getAvailableStrategies(strategyIds)\n\n if (activeStrategies.length === 0) {\n // Try fallback strategy if defined\n if (this.fallbackStrategy) {\n const fallback = await this.getAvailableStrategies([this.fallbackStrategy])\n if (fallback.length > 0) {\n activeStrategies.push(...fallback)\n }\n }\n }\n\n if (activeStrategies.length === 0) {\n return []\n }\n\n // Execute searches in parallel with graceful degradation\n const results = await Promise.allSettled(\n activeStrategies.map((strategy) => this.executeStrategySearch(strategy, query, options)),\n )\n\n // Collect successful results, log failures\n const allResults: SearchResult[] = []\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'fulfilled') {\n allResults.push(...result.value)\n } else {\n const strategy = activeStrategies[i]\n searchError('SearchService', 'Strategy search failed', {\n strategyId: strategy?.id,\n error: result.reason instanceof Error ? result.reason.message : result.reason,\n })\n }\n }\n\n // Merge and rank results\n const merged = mergeAndRankResults(allResults, this.mergeConfig)\n const scoped = filterResultsByOrganizationScope(merged, options)\n\n // Enrich results missing presenter or navigation metadata\n return this.enrichResultsWithPresenter(scoped, options.tenantId, options.organizationId)\n }\n\n /**\n * Recompute configured presenters at request time and fill missing presenter\n * or navigation data for unconfigured results.\n */\n private async enrichResultsWithPresenter(\n results: SearchResult[],\n tenantId: string,\n organizationId?: string | null,\n ): Promise<SearchResult[]> {\n // If no enricher configured, return as-is\n if (!this.presenterEnricher) return results\n\n try {\n return await this.presenterEnricher(results, tenantId, organizationId)\n } catch {\n // Enrichment failed, return results as-is\n return results\n }\n }\n\n /**\n * Index a record across all available strategies.\n *\n * @param record - Record to index\n */\n async index(record: IndexableRecord): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n if (strategies.length === 0) {\n return\n }\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => this.executeStrategyIndex(strategy, record)),\n )\n\n this.throwOnStrategyFailures('index', strategies, results, {\n entityId: record.entityId,\n recordId: record.recordId,\n })\n }\n\n /**\n * Delete a record from all strategies.\n *\n * @param entityId - Entity type identifier\n * @param recordId - Record primary key\n * @param tenantId - Tenant for isolation\n */\n async delete(entityId: string, recordId: string, tenantId: string): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => this.executeStrategyDelete(strategy, entityId, recordId, tenantId)),\n )\n\n this.throwOnStrategyFailures('delete', strategies, results, { entityId, recordId })\n }\n\n /**\n * Bulk index multiple records.\n *\n * @param records - Records to index\n */\n async bulkIndex(records: IndexableRecord[]): Promise<void> {\n if (records.length === 0) return\n\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => {\n if (strategy.bulkIndex) {\n return strategy.bulkIndex(records)\n }\n // Fallback to individual indexing\n return Promise.all(records.map((record) => this.executeStrategyIndex(strategy, record)))\n }),\n )\n\n // Collect failures and throw if any occurred\n const failures: Array<{ strategyId: string; error: string }> = []\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'rejected') {\n const strategy = strategies[i]\n const errorMessage = result.reason instanceof Error ? result.reason.message : result.reason\n failures.push({\n strategyId: strategy?.id || 'unknown',\n error: errorMessage,\n })\n searchError('SearchService', 'Strategy bulkIndex failed', {\n strategyId: strategy?.id,\n recordCount: records.length,\n error: errorMessage,\n })\n }\n }\n\n if (failures.length > 0) {\n throw new Error(\n `Bulk indexing failed for ${failures.length} strategy(ies): ${failures\n .map((f) => `${f.strategyId} (${f.error})`)\n .join(', ')}`\n )\n }\n }\n\n /**\n * Purge all records for an entity type.\n *\n * @param entityId - Entity type to purge\n * @param tenantId - Tenant for isolation\n */\n async purge(entityId: string, tenantId: string, organizationId?: string | null): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => {\n if (strategy.purge) {\n return strategy.purge(entityId, tenantId, organizationId)\n }\n return Promise.resolve()\n }),\n )\n\n this.throwOnStrategyFailures('purge', strategies, results, { entityId })\n }\n\n /**\n * Inspect the settled results of a per-strategy write operation, log every\n * rejection, and re-throw an aggregated error when any strategy failed.\n *\n * Write operations (index/delete/purge) must surface failures to the caller\n * so the queue worker re-throws and the job is retried. Swallowing rejections\n * here causes silent, permanent index gaps on transient failures such as\n * Postgres connection-pool exhaustion (issue #3103). Successful strategies\n * still commit their work; only the aggregated failure propagates.\n */\n private throwOnStrategyFailures(\n operation: 'index' | 'delete' | 'purge',\n strategies: SearchStrategy[],\n results: PromiseSettledResult<unknown>[],\n context: { entityId: string; recordId?: string },\n ): void {\n const failures: Array<{ strategyId: string; reason: unknown }> = []\n\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'rejected') {\n const strategy = strategies[i]\n failures.push({ strategyId: strategy?.id || 'unknown', reason: result.reason })\n searchError('SearchService', `Strategy ${operation} failed`, {\n strategyId: strategy?.id,\n entityId: context.entityId,\n recordId: context.recordId,\n error: result.reason instanceof Error ? result.reason.message : result.reason,\n })\n }\n }\n\n if (failures.length === 0) return\n\n const summary = `Search ${operation} failed for ${failures.length} strategy(ies): ${failures\n .map((failure) => {\n const message = failure.reason instanceof Error ? failure.reason.message : String(failure.reason)\n return `${failure.strategyId} (${message})`\n })\n .join(', ')}`\n\n throw new AggregateError(\n failures.map((failure) => failure.reason),\n summary,\n )\n }\n\n /**\n * Register a new strategy at runtime.\n *\n * @param strategy - Strategy to register\n */\n registerStrategy(strategy: SearchStrategy): void {\n this.strategies.set(strategy.id, strategy)\n this.availabilityCache.delete(strategy.id)\n this.availabilityInflight.delete(strategy.id)\n }\n\n /**\n * Unregister a strategy.\n *\n * @param strategyId - Strategy ID to remove\n */\n unregisterStrategy(strategyId: SearchStrategyId): void {\n this.strategies.delete(strategyId)\n this.availabilityCache.delete(strategyId)\n this.availabilityInflight.delete(strategyId)\n }\n\n /**\n * Invalidate the strategy availability cache.\n * Call after manual reconnects or env changes when callers must observe the\n * current backend state immediately rather than waiting for TTL expiry.\n */\n invalidateAvailabilityCache(strategyId?: SearchStrategyId): void {\n if (strategyId) {\n this.availabilityCache.delete(strategyId)\n this.availabilityInflight.delete(strategyId)\n return\n }\n this.availabilityCache.clear()\n this.availabilityInflight.clear()\n }\n\n /**\n * Get all registered strategy IDs.\n */\n getRegisteredStrategies(): SearchStrategyId[] {\n return Array.from(this.strategies.keys())\n }\n\n /**\n * Get a specific strategy by ID.\n *\n * @param strategyId - Strategy ID to retrieve\n * @returns The strategy if registered, undefined otherwise\n */\n getStrategy(strategyId: SearchStrategyId): SearchStrategy | undefined {\n return this.strategies.get(strategyId)\n }\n\n /**\n * Get the default strategies list.\n */\n getDefaultStrategies(): SearchStrategyId[] {\n return [...this.defaultStrategies]\n }\n\n /**\n * Check if a specific strategy is available.\n *\n * @param strategyId - Strategy ID to check\n */\n async isStrategyAvailable(strategyId: SearchStrategyId): Promise<boolean> {\n const strategy = this.strategies.get(strategyId)\n if (!strategy) return false\n return this.checkStrategyAvailability(strategy)\n }\n\n /**\n * Resolve a strategy's availability via the short-lived TTL cache.\n * Coalesces concurrent callers onto a single in-flight probe to avoid\n * thundering-herd on remote backends.\n */\n private async checkStrategyAvailability(strategy: SearchStrategy): Promise<boolean> {\n const now = Date.now()\n const cached = this.availabilityCache.get(strategy.id)\n if (cached && cached.expiresAt > now) return cached.value\n\n const inflight = this.availabilityInflight.get(strategy.id)\n if (inflight) return inflight\n\n const probe = (async () => {\n try {\n const value = await strategy.isAvailable()\n this.availabilityCache.set(strategy.id, {\n value,\n expiresAt: Date.now() + this.availabilityCacheTtlMs,\n })\n return value\n } catch {\n this.availabilityCache.set(strategy.id, {\n value: false,\n expiresAt: Date.now() + this.availabilityCacheTtlMs,\n })\n return false\n } finally {\n this.availabilityInflight.delete(strategy.id)\n }\n })()\n this.availabilityInflight.set(strategy.id, probe)\n return probe\n }\n\n /**\n * Get available strategies from the requested list.\n * Filters out strategies that are not registered or not available.\n * Probes run in parallel and reuse a short-lived per-strategy availability\n * cache, so hot paths pay the max latency of the slowest probe (or zero\n * when cached) instead of the sum of all probes.\n */\n private async getAvailableStrategies(ids?: SearchStrategyId[]): Promise<SearchStrategy[]> {\n const targetIds = ids ?? Array.from(this.strategies.keys())\n const candidates: SearchStrategy[] = []\n for (const id of targetIds) {\n const strategy = this.strategies.get(id)\n if (strategy) candidates.push(strategy)\n }\n\n const probes = await Promise.allSettled(\n candidates.map((strategy) => this.checkStrategyAvailability(strategy)),\n )\n\n const available: SearchStrategy[] = []\n for (let i = 0; i < probes.length; i++) {\n const probe = probes[i]\n if (probe.status === 'fulfilled' && probe.value) {\n available.push(candidates[i])\n }\n }\n\n return available.sort((a, b) => b.priority - a.priority)\n }\n\n /**\n * Execute search on a single strategy with error handling.\n */\n private async executeStrategySearch(\n strategy: SearchStrategy,\n query: string,\n options: SearchOptions,\n ): Promise<SearchResult[]> {\n await strategy.ensureReady()\n return strategy.search(query, options)\n }\n\n /**\n * Execute index on a single strategy with error handling.\n */\n private async executeStrategyIndex(\n strategy: SearchStrategy,\n record: IndexableRecord,\n ): Promise<void> {\n await strategy.ensureReady()\n return strategy.index(record)\n }\n\n /**\n * Execute delete on a single strategy with error handling.\n */\n private async executeStrategyDelete(\n strategy: SearchStrategy,\n entityId: string,\n recordId: string,\n tenantId: string,\n ): Promise<void> {\n await strategy.ensureReady()\n return strategy.delete(entityId, recordId, tenantId)\n }\n}\n"],
|
|
5
|
-
"mappings": "AAUA,SAAS,2BAA2B;AACpC,SAAS,mBAAmB;AAK5B,MAAM,uBAA0C;AAAA,EAC9C,mBAAmB;AACrB;AAOA,MAAM,qCAAqC;
|
|
4
|
+
"sourcesContent": ["import type {\n SearchStrategy,\n SearchStrategyId,\n SearchOptions,\n SearchResult,\n SearchServiceOptions,\n ResultMergeConfig,\n IndexableRecord,\n PresenterEnricherFn,\n} from './types'\nimport { mergeAndRankResults } from './lib/merger'\nimport { searchError } from './lib/debug'\n\n/**\n * Default merge configuration.\n */\nconst DEFAULT_MERGE_CONFIG: ResultMergeConfig = {\n duplicateHandling: 'highest_score',\n}\n\n/**\n * Cache TTL for strategy availability checks.\n * Short window so connectivity changes (Meilisearch up/down) propagate quickly,\n * long enough to skip per-request RTT to remote backends on hot paths.\n */\nconst STRATEGY_AVAILABILITY_CACHE_TTL_MS = 2_000\n\n/**\n * Maximum records indexed at once when bulkIndex falls back to per-record writes\n * for a strategy that has no bulkIndex implementation (currently the vector\n * strategy, whose index() performs an embedding-provider round trip per record).\n * A whole reindex page arrives in one bulkIndex call, so an unbounded fan-out\n * would burst hundreds of concurrent provider requests from a single job.\n */\nconst BULK_INDEX_FALLBACK_CONCURRENCY = 4\n\n/**\n * Map items through an async worker with a fixed number of in-flight calls.\n * Rejects with the first error, matching Promise.all semantics.\n */\nasync function mapWithConcurrency<T>(\n items: T[],\n limit: number,\n worker: (item: T) => Promise<void>,\n): Promise<void> {\n if (items.length === 0) return\n\n let nextIndex = 0\n const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {\n for (;;) {\n const currentIndex = nextIndex++\n if (currentIndex >= items.length) return\n await worker(items[currentIndex])\n }\n })\n\n await Promise.all(runners)\n}\n\nfunction normalizeOrganizationFilter(options: SearchOptions): string[] | null {\n const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''\n if (single) return [single]\n if (!Array.isArray(options.organizationIds)) return null\n\n const values = Array.from(new Set(\n options.organizationIds\n .map((value) => (typeof value === 'string' ? value.trim() : ''))\n .filter((value) => value.length > 0),\n ))\n return values\n}\n\nfunction filterResultsByOrganizationScope(results: SearchResult[], options: SearchOptions): SearchResult[] {\n const organizationIds = normalizeOrganizationFilter(options)\n if (!organizationIds) return results\n if (organizationIds.length === 0) return []\n\n const allowed = new Set(organizationIds)\n return results.filter((result) => {\n const organizationId = typeof result.organizationId === 'string' ? result.organizationId.trim() : ''\n return organizationId.length > 0 && allowed.has(organizationId)\n })\n}\n\n/**\n * SearchService orchestrates multiple search strategies, executing searches in parallel\n * and merging results using the RRF algorithm.\n *\n * Features:\n * - Parallel strategy execution for optimal performance\n * - Graceful degradation when strategies fail\n * - Result merging with configurable weights\n * - Strategy availability checking\n *\n * @example\n * ```typescript\n * const service = new SearchService({\n * strategies: [tokenStrategy, vectorStrategy, fulltextStrategy],\n * defaultStrategies: ['fulltext', 'vector', 'tokens'],\n * mergeConfig: {\n * duplicateHandling: 'highest_score',\n * strategyWeights: { fulltext: 1.2, vector: 1.0, tokens: 0.8 },\n * },\n * })\n *\n * const results = await service.search('john doe', { tenantId: 'tenant-123' })\n * ```\n */\nexport class SearchService {\n private readonly strategies: Map<SearchStrategyId, SearchStrategy>\n private readonly defaultStrategies: SearchStrategyId[]\n private readonly fallbackStrategy: SearchStrategyId | undefined\n private readonly mergeConfig: ResultMergeConfig\n private readonly presenterEnricher?: PresenterEnricherFn\n private readonly availabilityCache = new Map<SearchStrategyId, { value: boolean; expiresAt: number }>()\n private readonly availabilityInflight = new Map<SearchStrategyId, Promise<boolean>>()\n private readonly availabilityCacheTtlMs: number\n\n constructor(options: SearchServiceOptions = {}) {\n this.strategies = new Map()\n for (const strategy of options.strategies ?? []) {\n this.strategies.set(strategy.id, strategy)\n }\n this.defaultStrategies = options.defaultStrategies ?? ['tokens']\n this.fallbackStrategy = options.fallbackStrategy\n this.mergeConfig = options.mergeConfig ?? DEFAULT_MERGE_CONFIG\n this.presenterEnricher = options.presenterEnricher\n this.availabilityCacheTtlMs = options.availabilityCacheTtlMs ?? STRATEGY_AVAILABILITY_CACHE_TTL_MS\n }\n\n /**\n * Get all registered strategies.\n */\n getStrategies(): SearchStrategy[] {\n return Array.from(this.strategies.values())\n }\n\n /**\n * Execute a search query across configured strategies.\n *\n * @param query - Search query string\n * @param options - Search options with tenant, filters, etc.\n * @returns Merged and ranked search results\n */\n async search(query: string, options: SearchOptions): Promise<SearchResult[]> {\n const organizationIds = normalizeOrganizationFilter(options)\n if (organizationIds && organizationIds.length === 0) {\n return []\n }\n\n const strategyIds = options.strategies ?? this.defaultStrategies\n const activeStrategies = await this.getAvailableStrategies(strategyIds)\n\n if (activeStrategies.length === 0) {\n // Try fallback strategy if defined\n if (this.fallbackStrategy) {\n const fallback = await this.getAvailableStrategies([this.fallbackStrategy])\n if (fallback.length > 0) {\n activeStrategies.push(...fallback)\n }\n }\n }\n\n if (activeStrategies.length === 0) {\n return []\n }\n\n // Execute searches in parallel with graceful degradation\n const results = await Promise.allSettled(\n activeStrategies.map((strategy) => this.executeStrategySearch(strategy, query, options)),\n )\n\n // Collect successful results, log failures\n const allResults: SearchResult[] = []\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'fulfilled') {\n allResults.push(...result.value)\n } else {\n const strategy = activeStrategies[i]\n searchError('SearchService', 'Strategy search failed', {\n strategyId: strategy?.id,\n error: result.reason instanceof Error ? result.reason.message : result.reason,\n })\n }\n }\n\n // Merge and rank results\n const merged = mergeAndRankResults(allResults, this.mergeConfig)\n const scoped = filterResultsByOrganizationScope(merged, options)\n\n // Enrich results missing presenter or navigation metadata\n return this.enrichResultsWithPresenter(scoped, options.tenantId, options.organizationId)\n }\n\n /**\n * Recompute configured presenters at request time and fill missing presenter\n * or navigation data for unconfigured results.\n */\n private async enrichResultsWithPresenter(\n results: SearchResult[],\n tenantId: string,\n organizationId?: string | null,\n ): Promise<SearchResult[]> {\n // If no enricher configured, return as-is\n if (!this.presenterEnricher) return results\n\n try {\n return await this.presenterEnricher(results, tenantId, organizationId)\n } catch {\n // Enrichment failed, return results as-is\n return results\n }\n }\n\n /**\n * Index a record across all available strategies.\n *\n * @param record - Record to index\n */\n async index(record: IndexableRecord): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n if (strategies.length === 0) {\n return\n }\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => this.executeStrategyIndex(strategy, record)),\n )\n\n this.throwOnStrategyFailures('index', strategies, results, {\n entityId: record.entityId,\n recordId: record.recordId,\n })\n }\n\n /**\n * Delete a record from all strategies.\n *\n * @param entityId - Entity type identifier\n * @param recordId - Record primary key\n * @param tenantId - Tenant for isolation\n */\n async delete(entityId: string, recordId: string, tenantId: string): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => this.executeStrategyDelete(strategy, entityId, recordId, tenantId)),\n )\n\n this.throwOnStrategyFailures('delete', strategies, results, { entityId, recordId })\n }\n\n /**\n * Bulk index multiple records.\n *\n * @param records - Records to index\n */\n async bulkIndex(records: IndexableRecord[]): Promise<void> {\n if (records.length === 0) return\n\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => {\n if (strategy.bulkIndex) {\n return strategy.bulkIndex(records)\n }\n // Fallback to individual indexing, bounded so a strategy without a batch\n // implementation cannot turn one batch job into hundreds of concurrent writes.\n return mapWithConcurrency(records, BULK_INDEX_FALLBACK_CONCURRENCY, (record) =>\n this.executeStrategyIndex(strategy, record),\n )\n }),\n )\n\n // Collect failures and throw if any occurred\n const failures: Array<{ strategyId: string; error: string }> = []\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'rejected') {\n const strategy = strategies[i]\n const errorMessage = result.reason instanceof Error ? result.reason.message : result.reason\n failures.push({\n strategyId: strategy?.id || 'unknown',\n error: errorMessage,\n })\n searchError('SearchService', 'Strategy bulkIndex failed', {\n strategyId: strategy?.id,\n recordCount: records.length,\n error: errorMessage,\n })\n }\n }\n\n if (failures.length > 0) {\n throw new Error(\n `Bulk indexing failed for ${failures.length} strategy(ies): ${failures\n .map((f) => `${f.strategyId} (${f.error})`)\n .join(', ')}`\n )\n }\n }\n\n /**\n * Purge all records for an entity type.\n *\n * @param entityId - Entity type to purge\n * @param tenantId - Tenant for isolation\n */\n async purge(entityId: string, tenantId: string, organizationId?: string | null): Promise<void> {\n const strategies = await this.getAvailableStrategies()\n\n const results = await Promise.allSettled(\n strategies.map((strategy) => {\n if (strategy.purge) {\n return strategy.purge(entityId, tenantId, organizationId)\n }\n return Promise.resolve()\n }),\n )\n\n this.throwOnStrategyFailures('purge', strategies, results, { entityId })\n }\n\n /**\n * Inspect the settled results of a per-strategy write operation, log every\n * rejection, and re-throw an aggregated error when any strategy failed.\n *\n * Write operations (index/delete/purge) must surface failures to the caller\n * so the queue worker re-throws and the job is retried. Swallowing rejections\n * here causes silent, permanent index gaps on transient failures such as\n * Postgres connection-pool exhaustion (issue #3103). Successful strategies\n * still commit their work; only the aggregated failure propagates.\n */\n private throwOnStrategyFailures(\n operation: 'index' | 'delete' | 'purge',\n strategies: SearchStrategy[],\n results: PromiseSettledResult<unknown>[],\n context: { entityId: string; recordId?: string },\n ): void {\n const failures: Array<{ strategyId: string; reason: unknown }> = []\n\n for (let i = 0; i < results.length; i++) {\n const result = results[i]\n if (result.status === 'rejected') {\n const strategy = strategies[i]\n failures.push({ strategyId: strategy?.id || 'unknown', reason: result.reason })\n searchError('SearchService', `Strategy ${operation} failed`, {\n strategyId: strategy?.id,\n entityId: context.entityId,\n recordId: context.recordId,\n error: result.reason instanceof Error ? result.reason.message : result.reason,\n })\n }\n }\n\n if (failures.length === 0) return\n\n const summary = `Search ${operation} failed for ${failures.length} strategy(ies): ${failures\n .map((failure) => {\n const message = failure.reason instanceof Error ? failure.reason.message : String(failure.reason)\n return `${failure.strategyId} (${message})`\n })\n .join(', ')}`\n\n throw new AggregateError(\n failures.map((failure) => failure.reason),\n summary,\n )\n }\n\n /**\n * Register a new strategy at runtime.\n *\n * @param strategy - Strategy to register\n */\n registerStrategy(strategy: SearchStrategy): void {\n this.strategies.set(strategy.id, strategy)\n this.availabilityCache.delete(strategy.id)\n this.availabilityInflight.delete(strategy.id)\n }\n\n /**\n * Unregister a strategy.\n *\n * @param strategyId - Strategy ID to remove\n */\n unregisterStrategy(strategyId: SearchStrategyId): void {\n this.strategies.delete(strategyId)\n this.availabilityCache.delete(strategyId)\n this.availabilityInflight.delete(strategyId)\n }\n\n /**\n * Invalidate the strategy availability cache.\n * Call after manual reconnects or env changes when callers must observe the\n * current backend state immediately rather than waiting for TTL expiry.\n */\n invalidateAvailabilityCache(strategyId?: SearchStrategyId): void {\n if (strategyId) {\n this.availabilityCache.delete(strategyId)\n this.availabilityInflight.delete(strategyId)\n return\n }\n this.availabilityCache.clear()\n this.availabilityInflight.clear()\n }\n\n /**\n * Get all registered strategy IDs.\n */\n getRegisteredStrategies(): SearchStrategyId[] {\n return Array.from(this.strategies.keys())\n }\n\n /**\n * Get a specific strategy by ID.\n *\n * @param strategyId - Strategy ID to retrieve\n * @returns The strategy if registered, undefined otherwise\n */\n getStrategy(strategyId: SearchStrategyId): SearchStrategy | undefined {\n return this.strategies.get(strategyId)\n }\n\n /**\n * Get the default strategies list.\n */\n getDefaultStrategies(): SearchStrategyId[] {\n return [...this.defaultStrategies]\n }\n\n /**\n * Check if a specific strategy is available.\n *\n * @param strategyId - Strategy ID to check\n */\n async isStrategyAvailable(strategyId: SearchStrategyId): Promise<boolean> {\n const strategy = this.strategies.get(strategyId)\n if (!strategy) return false\n return this.checkStrategyAvailability(strategy)\n }\n\n /**\n * Resolve a strategy's availability via the short-lived TTL cache.\n * Coalesces concurrent callers onto a single in-flight probe to avoid\n * thundering-herd on remote backends.\n */\n private async checkStrategyAvailability(strategy: SearchStrategy): Promise<boolean> {\n const now = Date.now()\n const cached = this.availabilityCache.get(strategy.id)\n if (cached && cached.expiresAt > now) return cached.value\n\n const inflight = this.availabilityInflight.get(strategy.id)\n if (inflight) return inflight\n\n const probe = (async () => {\n try {\n const value = await strategy.isAvailable()\n this.availabilityCache.set(strategy.id, {\n value,\n expiresAt: Date.now() + this.availabilityCacheTtlMs,\n })\n return value\n } catch {\n this.availabilityCache.set(strategy.id, {\n value: false,\n expiresAt: Date.now() + this.availabilityCacheTtlMs,\n })\n return false\n } finally {\n this.availabilityInflight.delete(strategy.id)\n }\n })()\n this.availabilityInflight.set(strategy.id, probe)\n return probe\n }\n\n /**\n * Get available strategies from the requested list.\n * Filters out strategies that are not registered or not available.\n * Probes run in parallel and reuse a short-lived per-strategy availability\n * cache, so hot paths pay the max latency of the slowest probe (or zero\n * when cached) instead of the sum of all probes.\n */\n private async getAvailableStrategies(ids?: SearchStrategyId[]): Promise<SearchStrategy[]> {\n const targetIds = ids ?? Array.from(this.strategies.keys())\n const candidates: SearchStrategy[] = []\n for (const id of targetIds) {\n const strategy = this.strategies.get(id)\n if (strategy) candidates.push(strategy)\n }\n\n const probes = await Promise.allSettled(\n candidates.map((strategy) => this.checkStrategyAvailability(strategy)),\n )\n\n const available: SearchStrategy[] = []\n for (let i = 0; i < probes.length; i++) {\n const probe = probes[i]\n if (probe.status === 'fulfilled' && probe.value) {\n available.push(candidates[i])\n }\n }\n\n return available.sort((a, b) => b.priority - a.priority)\n }\n\n /**\n * Execute search on a single strategy with error handling.\n */\n private async executeStrategySearch(\n strategy: SearchStrategy,\n query: string,\n options: SearchOptions,\n ): Promise<SearchResult[]> {\n await strategy.ensureReady()\n return strategy.search(query, options)\n }\n\n /**\n * Execute index on a single strategy with error handling.\n */\n private async executeStrategyIndex(\n strategy: SearchStrategy,\n record: IndexableRecord,\n ): Promise<void> {\n await strategy.ensureReady()\n return strategy.index(record)\n }\n\n /**\n * Execute delete on a single strategy with error handling.\n */\n private async executeStrategyDelete(\n strategy: SearchStrategy,\n entityId: string,\n recordId: string,\n tenantId: string,\n ): Promise<void> {\n await strategy.ensureReady()\n return strategy.delete(entityId, recordId, tenantId)\n }\n}\n"],
|
|
5
|
+
"mappings": "AAUA,SAAS,2BAA2B;AACpC,SAAS,mBAAmB;AAK5B,MAAM,uBAA0C;AAAA,EAC9C,mBAAmB;AACrB;AAOA,MAAM,qCAAqC;AAS3C,MAAM,kCAAkC;AAMxC,eAAe,mBACb,OACA,OACA,QACe;AACf,MAAI,MAAM,WAAW,EAAG;AAExB,MAAI,YAAY;AAChB,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,OAAO,MAAM,MAAM,EAAE,GAAG,YAAY;AAChF,eAAS;AACP,YAAM,eAAe;AACrB,UAAI,gBAAgB,MAAM,OAAQ;AAClC,YAAM,OAAO,MAAM,YAAY,CAAC;AAAA,IAClC;AAAA,EACF,CAAC;AAED,QAAM,QAAQ,IAAI,OAAO;AAC3B;AAEA,SAAS,4BAA4B,SAAyC;AAC5E,QAAM,SAAS,OAAO,QAAQ,mBAAmB,WAAW,QAAQ,eAAe,KAAK,IAAI;AAC5F,MAAI,OAAQ,QAAO,CAAC,MAAM;AAC1B,MAAI,CAAC,MAAM,QAAQ,QAAQ,eAAe,EAAG,QAAO;AAEpD,QAAM,SAAS,MAAM,KAAK,IAAI;AAAA,IAC5B,QAAQ,gBACL,IAAI,CAAC,UAAW,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI,EAAG,EAC9D,OAAO,CAAC,UAAU,MAAM,SAAS,CAAC;AAAA,EACvC,CAAC;AACD,SAAO;AACT;AAEA,SAAS,iCAAiC,SAAyB,SAAwC;AACzG,QAAM,kBAAkB,4BAA4B,OAAO;AAC3D,MAAI,CAAC,gBAAiB,QAAO;AAC7B,MAAI,gBAAgB,WAAW,EAAG,QAAO,CAAC;AAE1C,QAAM,UAAU,IAAI,IAAI,eAAe;AACvC,SAAO,QAAQ,OAAO,CAAC,WAAW;AAChC,UAAM,iBAAiB,OAAO,OAAO,mBAAmB,WAAW,OAAO,eAAe,KAAK,IAAI;AAClG,WAAO,eAAe,SAAS,KAAK,QAAQ,IAAI,cAAc;AAAA,EAChE,CAAC;AACH;AA0BO,MAAM,cAAc;AAAA,EAUzB,YAAY,UAAgC,CAAC,GAAG;AAJhD,SAAiB,oBAAoB,oBAAI,IAA6D;AACtG,SAAiB,uBAAuB,oBAAI,IAAwC;AAIlF,SAAK,aAAa,oBAAI,IAAI;AAC1B,eAAW,YAAY,QAAQ,cAAc,CAAC,GAAG;AAC/C,WAAK,WAAW,IAAI,SAAS,IAAI,QAAQ;AAAA,IAC3C;AACA,SAAK,oBAAoB,QAAQ,qBAAqB,CAAC,QAAQ;AAC/D,SAAK,mBAAmB,QAAQ;AAChC,SAAK,cAAc,QAAQ,eAAe;AAC1C,SAAK,oBAAoB,QAAQ;AACjC,SAAK,yBAAyB,QAAQ,0BAA0B;AAAA,EAClE;AAAA;AAAA;AAAA;AAAA,EAKA,gBAAkC;AAChC,WAAO,MAAM,KAAK,KAAK,WAAW,OAAO,CAAC;AAAA,EAC5C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,OAAO,OAAe,SAAiD;AAC3E,UAAM,kBAAkB,4BAA4B,OAAO;AAC3D,QAAI,mBAAmB,gBAAgB,WAAW,GAAG;AACnD,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,cAAc,QAAQ,cAAc,KAAK;AAC/C,UAAM,mBAAmB,MAAM,KAAK,uBAAuB,WAAW;AAEtE,QAAI,iBAAiB,WAAW,GAAG;AAEjC,UAAI,KAAK,kBAAkB;AACzB,cAAM,WAAW,MAAM,KAAK,uBAAuB,CAAC,KAAK,gBAAgB,CAAC;AAC1E,YAAI,SAAS,SAAS,GAAG;AACvB,2BAAiB,KAAK,GAAG,QAAQ;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AAEA,QAAI,iBAAiB,WAAW,GAAG;AACjC,aAAO,CAAC;AAAA,IACV;AAGA,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,iBAAiB,IAAI,CAAC,aAAa,KAAK,sBAAsB,UAAU,OAAO,OAAO,CAAC;AAAA,IACzF;AAGA,UAAM,aAA6B,CAAC;AACpC,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,SAAS,QAAQ,CAAC;AACxB,UAAI,OAAO,WAAW,aAAa;AACjC,mBAAW,KAAK,GAAG,OAAO,KAAK;AAAA,MACjC,OAAO;AACL,cAAM,WAAW,iBAAiB,CAAC;AACnC,oBAAY,iBAAiB,0BAA0B;AAAA,UACrD,YAAY,UAAU;AAAA,UACtB,OAAO,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO;AAAA,QACzE,CAAC;AAAA,MACH;AAAA,IACF;AAGA,UAAM,SAAS,oBAAoB,YAAY,KAAK,WAAW;AAC/D,UAAM,SAAS,iCAAiC,QAAQ,OAAO;AAG/D,WAAO,KAAK,2BAA2B,QAAQ,QAAQ,UAAU,QAAQ,cAAc;AAAA,EACzF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,2BACZ,SACA,UACA,gBACyB;AAEzB,QAAI,CAAC,KAAK,kBAAmB,QAAO;AAEpC,QAAI;AACF,aAAO,MAAM,KAAK,kBAAkB,SAAS,UAAU,cAAc;AAAA,IACvE,QAAQ;AAEN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,MAAM,QAAwC;AAClD,UAAM,aAAa,MAAM,KAAK,uBAAuB;AAErD,QAAI,WAAW,WAAW,GAAG;AAC3B;AAAA,IACF;AAEA,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,WAAW,IAAI,CAAC,aAAa,KAAK,qBAAqB,UAAU,MAAM,CAAC;AAAA,IAC1E;AAEA,SAAK,wBAAwB,SAAS,YAAY,SAAS;AAAA,MACzD,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,OAAO,UAAkB,UAAkB,UAAiC;AAChF,UAAM,aAAa,MAAM,KAAK,uBAAuB;AAErD,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,WAAW,IAAI,CAAC,aAAa,KAAK,sBAAsB,UAAU,UAAU,UAAU,QAAQ,CAAC;AAAA,IACjG;AAEA,SAAK,wBAAwB,UAAU,YAAY,SAAS,EAAE,UAAU,SAAS,CAAC;AAAA,EACpF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,UAAU,SAA2C;AACzD,QAAI,QAAQ,WAAW,EAAG;AAE1B,UAAM,aAAa,MAAM,KAAK,uBAAuB;AAErD,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,WAAW,IAAI,CAAC,aAAa;AAC3B,YAAI,SAAS,WAAW;AACtB,iBAAO,SAAS,UAAU,OAAO;AAAA,QACnC;AAGA,eAAO;AAAA,UAAmB;AAAA,UAAS;AAAA,UAAiC,CAAC,WACnE,KAAK,qBAAqB,UAAU,MAAM;AAAA,QAC5C;AAAA,MACF,CAAC;AAAA,IACH;AAGA,UAAM,WAAyD,CAAC;AAChE,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,SAAS,QAAQ,CAAC;AACxB,UAAI,OAAO,WAAW,YAAY;AAChC,cAAM,WAAW,WAAW,CAAC;AAC7B,cAAM,eAAe,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO;AACrF,iBAAS,KAAK;AAAA,UACZ,YAAY,UAAU,MAAM;AAAA,UAC5B,OAAO;AAAA,QACT,CAAC;AACD,oBAAY,iBAAiB,6BAA6B;AAAA,UACxD,YAAY,UAAU;AAAA,UACtB,aAAa,QAAQ;AAAA,UACrB,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,IAAI;AAAA,QACR,4BAA4B,SAAS,MAAM,mBAAmB,SAC3D,IAAI,CAAC,MAAM,GAAG,EAAE,UAAU,KAAK,EAAE,KAAK,GAAG,EACzC,KAAK,IAAI,CAAC;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,MAAM,UAAkB,UAAkB,gBAA+C;AAC7F,UAAM,aAAa,MAAM,KAAK,uBAAuB;AAErD,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,WAAW,IAAI,CAAC,aAAa;AAC3B,YAAI,SAAS,OAAO;AAClB,iBAAO,SAAS,MAAM,UAAU,UAAU,cAAc;AAAA,QAC1D;AACA,eAAO,QAAQ,QAAQ;AAAA,MACzB,CAAC;AAAA,IACH;AAEA,SAAK,wBAAwB,SAAS,YAAY,SAAS,EAAE,SAAS,CAAC;AAAA,EACzE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYQ,wBACN,WACA,YACA,SACA,SACM;AACN,UAAM,WAA2D,CAAC;AAElE,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,SAAS,QAAQ,CAAC;AACxB,UAAI,OAAO,WAAW,YAAY;AAChC,cAAM,WAAW,WAAW,CAAC;AAC7B,iBAAS,KAAK,EAAE,YAAY,UAAU,MAAM,WAAW,QAAQ,OAAO,OAAO,CAAC;AAC9E,oBAAY,iBAAiB,YAAY,SAAS,WAAW;AAAA,UAC3D,YAAY,UAAU;AAAA,UACtB,UAAU,QAAQ;AAAA,UAClB,UAAU,QAAQ;AAAA,UAClB,OAAO,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO;AAAA,QACzE,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,SAAS,WAAW,EAAG;AAE3B,UAAM,UAAU,UAAU,SAAS,eAAe,SAAS,MAAM,mBAAmB,SACjF,IAAI,CAAC,YAAY;AAChB,YAAM,UAAU,QAAQ,kBAAkB,QAAQ,QAAQ,OAAO,UAAU,OAAO,QAAQ,MAAM;AAChG,aAAO,GAAG,QAAQ,UAAU,KAAK,OAAO;AAAA,IAC1C,CAAC,EACA,KAAK,IAAI,CAAC;AAEb,UAAM,IAAI;AAAA,MACR,SAAS,IAAI,CAAC,YAAY,QAAQ,MAAM;AAAA,MACxC;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,iBAAiB,UAAgC;AAC/C,SAAK,WAAW,IAAI,SAAS,IAAI,QAAQ;AACzC,SAAK,kBAAkB,OAAO,SAAS,EAAE;AACzC,SAAK,qBAAqB,OAAO,SAAS,EAAE;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,mBAAmB,YAAoC;AACrD,SAAK,WAAW,OAAO,UAAU;AACjC,SAAK,kBAAkB,OAAO,UAAU;AACxC,SAAK,qBAAqB,OAAO,UAAU;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,4BAA4B,YAAqC;AAC/D,QAAI,YAAY;AACd,WAAK,kBAAkB,OAAO,UAAU;AACxC,WAAK,qBAAqB,OAAO,UAAU;AAC3C;AAAA,IACF;AACA,SAAK,kBAAkB,MAAM;AAC7B,SAAK,qBAAqB,MAAM;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA,EAKA,0BAA8C;AAC5C,WAAO,MAAM,KAAK,KAAK,WAAW,KAAK,CAAC;AAAA,EAC1C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,YAAY,YAA0D;AACpE,WAAO,KAAK,WAAW,IAAI,UAAU;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKA,uBAA2C;AACzC,WAAO,CAAC,GAAG,KAAK,iBAAiB;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,oBAAoB,YAAgD;AACxE,UAAM,WAAW,KAAK,WAAW,IAAI,UAAU;AAC/C,QAAI,CAAC,SAAU,QAAO;AACtB,WAAO,KAAK,0BAA0B,QAAQ;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,0BAA0B,UAA4C;AAClF,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,SAAS,KAAK,kBAAkB,IAAI,SAAS,EAAE;AACrD,QAAI,UAAU,OAAO,YAAY,IAAK,QAAO,OAAO;AAEpD,UAAM,WAAW,KAAK,qBAAqB,IAAI,SAAS,EAAE;AAC1D,QAAI,SAAU,QAAO;AAErB,UAAM,SAAS,YAAY;AACzB,UAAI;AACF,cAAM,QAAQ,MAAM,SAAS,YAAY;AACzC,aAAK,kBAAkB,IAAI,SAAS,IAAI;AAAA,UACtC;AAAA,UACA,WAAW,KAAK,IAAI,IAAI,KAAK;AAAA,QAC/B,CAAC;AACD,eAAO;AAAA,MACT,QAAQ;AACN,aAAK,kBAAkB,IAAI,SAAS,IAAI;AAAA,UACtC,OAAO;AAAA,UACP,WAAW,KAAK,IAAI,IAAI,KAAK;AAAA,QAC/B,CAAC;AACD,eAAO;AAAA,MACT,UAAE;AACA,aAAK,qBAAqB,OAAO,SAAS,EAAE;AAAA,MAC9C;AAAA,IACF,GAAG;AACH,SAAK,qBAAqB,IAAI,SAAS,IAAI,KAAK;AAChD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAc,uBAAuB,KAAqD;AACxF,UAAM,YAAY,OAAO,MAAM,KAAK,KAAK,WAAW,KAAK,CAAC;AAC1D,UAAM,aAA+B,CAAC;AACtC,eAAW,MAAM,WAAW;AAC1B,YAAM,WAAW,KAAK,WAAW,IAAI,EAAE;AACvC,UAAI,SAAU,YAAW,KAAK,QAAQ;AAAA,IACxC;AAEA,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,WAAW,IAAI,CAAC,aAAa,KAAK,0BAA0B,QAAQ,CAAC;AAAA,IACvE;AAEA,UAAM,YAA8B,CAAC;AACrC,aAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,YAAM,QAAQ,OAAO,CAAC;AACtB,UAAI,MAAM,WAAW,eAAe,MAAM,OAAO;AAC/C,kBAAU,KAAK,WAAW,CAAC,CAAC;AAAA,MAC9B;AAAA,IACF;AAEA,WAAO,UAAU,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBACZ,UACA,OACA,SACyB;AACzB,UAAM,SAAS,YAAY;AAC3B,WAAO,SAAS,OAAO,OAAO,OAAO;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,qBACZ,UACA,QACe;AACf,UAAM,SAAS,YAAY;AAC3B,WAAO,SAAS,MAAM,MAAM;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,sBACZ,UACA,UACA,UACA,UACe;AACf,UAAM,SAAS,YAAY;AAC3B,WAAO,SAAS,OAAO,UAAU,UAAU,QAAQ;AAAA,EACrD;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/search",
|
|
3
|
-
"version": "0.6.8-develop.
|
|
3
|
+
"version": "0.6.8-develop.7100.1.fbf66fca35",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -127,9 +127,9 @@
|
|
|
127
127
|
"zod": "^4.4.3"
|
|
128
128
|
},
|
|
129
129
|
"peerDependencies": {
|
|
130
|
-
"@open-mercato/core": "0.6.8-develop.
|
|
131
|
-
"@open-mercato/queue": "0.6.8-develop.
|
|
132
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
130
|
+
"@open-mercato/core": "0.6.8-develop.7100.1.fbf66fca35",
|
|
131
|
+
"@open-mercato/queue": "0.6.8-develop.7100.1.fbf66fca35",
|
|
132
|
+
"@open-mercato/shared": "0.6.8-develop.7100.1.fbf66fca35"
|
|
133
133
|
},
|
|
134
134
|
"devDependencies": {
|
|
135
135
|
"@types/jest": "^30.0.0",
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { SearchIndexer } from '../indexer/search-indexer'
|
|
2
|
+
import type { SearchModuleConfig } from '../types'
|
|
3
|
+
import type { QueryEngine, QueryResult } from '@open-mercato/shared/lib/query/types'
|
|
4
|
+
|
|
5
|
+
describe('SearchIndexer.indexRecordsById', () => {
|
|
6
|
+
const moduleConfigs: SearchModuleConfig[] = [
|
|
7
|
+
{
|
|
8
|
+
entities: [
|
|
9
|
+
{
|
|
10
|
+
entityId: 'test:entity',
|
|
11
|
+
enabled: true,
|
|
12
|
+
formatResult: async (ctx) => ({ title: String(ctx.record.name ?? ctx.record.id) }),
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
entityId: 'test:other',
|
|
16
|
+
enabled: true,
|
|
17
|
+
formatResult: async (ctx) => ({ title: String(ctx.record.name ?? ctx.record.id) }),
|
|
18
|
+
},
|
|
19
|
+
],
|
|
20
|
+
},
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
function makeQueryEngine(recordsByEntity: Record<string, Record<string, unknown>[]>): QueryEngine {
|
|
24
|
+
return {
|
|
25
|
+
query: jest.fn(async (entity, opts) => {
|
|
26
|
+
const wantedId = (opts?.filters as { id?: string } | undefined)?.id
|
|
27
|
+
const items = (recordsByEntity[entity as string] ?? []).filter((r) => r.id === wantedId)
|
|
28
|
+
return { items, total: items.length } as QueryResult
|
|
29
|
+
}),
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
it('writes N queued records through exactly one bulkIndex call, not N', async () => {
|
|
34
|
+
const records = [
|
|
35
|
+
{ id: 'rec-1', name: 'Alpha' },
|
|
36
|
+
{ id: 'rec-2', name: 'Beta' },
|
|
37
|
+
{ id: 'rec-3', name: 'Gamma' },
|
|
38
|
+
]
|
|
39
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
40
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
|
|
41
|
+
queryEngine: makeQueryEngine({ 'test:entity': records }),
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
const result = await indexer.indexRecordsById({
|
|
45
|
+
items: [
|
|
46
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
47
|
+
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
48
|
+
{ entityId: 'test:entity', recordId: 'rec-3' },
|
|
49
|
+
],
|
|
50
|
+
tenantId: 'tenant-123',
|
|
51
|
+
organizationId: 'org-456',
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
|
|
55
|
+
expect(searchService.bulkIndex).toHaveBeenCalledWith([
|
|
56
|
+
expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-1', tenantId: 'tenant-123', organizationId: 'org-456' }),
|
|
57
|
+
expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-2', tenantId: 'tenant-123', organizationId: 'org-456' }),
|
|
58
|
+
expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-3', tenantId: 'tenant-123', organizationId: 'org-456' }),
|
|
59
|
+
])
|
|
60
|
+
expect(result).toEqual({ indexed: 3, skipped: 0 })
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
it('collapses records spanning multiple entities into one bulkIndex call', async () => {
|
|
64
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
65
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
|
|
66
|
+
queryEngine: makeQueryEngine({
|
|
67
|
+
'test:entity': [{ id: 'rec-1', name: 'Alpha' }],
|
|
68
|
+
'test:other': [{ id: 'other-1', name: 'Delta' }],
|
|
69
|
+
}),
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
const result = await indexer.indexRecordsById({
|
|
73
|
+
items: [
|
|
74
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
75
|
+
{ entityId: 'test:other', recordId: 'other-1' },
|
|
76
|
+
],
|
|
77
|
+
tenantId: 'tenant-123',
|
|
78
|
+
organizationId: null,
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
|
|
82
|
+
expect(searchService.bulkIndex).toHaveBeenCalledWith([
|
|
83
|
+
expect.objectContaining({ entityId: 'test:entity', recordId: 'rec-1' }),
|
|
84
|
+
expect.objectContaining({ entityId: 'test:other', recordId: 'other-1' }),
|
|
85
|
+
])
|
|
86
|
+
expect(result).toEqual({ indexed: 2, skipped: 0 })
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
it('loads each record with custom fields and without triggering auto-reindex', async () => {
|
|
90
|
+
const queryEngine = makeQueryEngine({ 'test:entity': [{ id: 'rec-1', name: 'Alpha' }] })
|
|
91
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
92
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
|
|
93
|
+
|
|
94
|
+
await indexer.indexRecordsById({
|
|
95
|
+
items: [{ entityId: 'test:entity', recordId: 'rec-1' }],
|
|
96
|
+
tenantId: 'tenant-123',
|
|
97
|
+
organizationId: 'org-456',
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
expect(queryEngine.query).toHaveBeenCalledWith(
|
|
101
|
+
'test:entity',
|
|
102
|
+
expect.objectContaining({
|
|
103
|
+
tenantId: 'tenant-123',
|
|
104
|
+
organizationId: 'org-456',
|
|
105
|
+
filters: { id: 'rec-1' },
|
|
106
|
+
includeCustomFields: true,
|
|
107
|
+
skipAutoReindex: true,
|
|
108
|
+
}),
|
|
109
|
+
)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
it('skips records that no longer exist without failing the batch write', async () => {
|
|
113
|
+
const records = [{ id: 'rec-1', name: 'Alpha' }]
|
|
114
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
115
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
|
|
116
|
+
queryEngine: makeQueryEngine({ 'test:entity': records }),
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
const result = await indexer.indexRecordsById({
|
|
120
|
+
items: [
|
|
121
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
122
|
+
{ entityId: 'test:entity', recordId: 'missing' },
|
|
123
|
+
],
|
|
124
|
+
tenantId: 'tenant-123',
|
|
125
|
+
organizationId: null,
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
|
|
129
|
+
expect(result).toEqual({ indexed: 1, skipped: 1 })
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
it('keeps indexing the batch when loading one record throws', async () => {
|
|
133
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
134
|
+
const queryEngine: QueryEngine = {
|
|
135
|
+
query: jest.fn(async (_entity, opts) => {
|
|
136
|
+
const wantedId = (opts?.filters as { id?: string } | undefined)?.id
|
|
137
|
+
if (wantedId === 'boom') throw new Error('connection lost')
|
|
138
|
+
return { items: [{ id: wantedId, name: 'Alpha' }], total: 1 } as QueryResult
|
|
139
|
+
}),
|
|
140
|
+
}
|
|
141
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
|
|
142
|
+
|
|
143
|
+
const result = await indexer.indexRecordsById({
|
|
144
|
+
items: [
|
|
145
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
146
|
+
{ entityId: 'test:entity', recordId: 'boom' },
|
|
147
|
+
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
148
|
+
],
|
|
149
|
+
tenantId: 'tenant-123',
|
|
150
|
+
organizationId: null,
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
expect(searchService.bulkIndex).toHaveBeenCalledTimes(1)
|
|
154
|
+
expect(searchService.bulkIndex).toHaveBeenCalledWith([
|
|
155
|
+
expect.objectContaining({ recordId: 'rec-1' }),
|
|
156
|
+
expect.objectContaining({ recordId: 'rec-2' }),
|
|
157
|
+
])
|
|
158
|
+
expect(result).toEqual({ indexed: 2, skipped: 1 })
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
it('propagates a bulkIndex failure so the queue can retry the job', async () => {
|
|
162
|
+
const searchService = { bulkIndex: jest.fn().mockRejectedValue(new Error('meilisearch unavailable')) }
|
|
163
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
|
|
164
|
+
queryEngine: makeQueryEngine({ 'test:entity': [{ id: 'rec-1', name: 'Alpha' }] }),
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
await expect(
|
|
168
|
+
indexer.indexRecordsById({
|
|
169
|
+
items: [{ entityId: 'test:entity', recordId: 'rec-1' }],
|
|
170
|
+
tenantId: 'tenant-123',
|
|
171
|
+
organizationId: null,
|
|
172
|
+
}),
|
|
173
|
+
).rejects.toThrow('meilisearch unavailable')
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
it('skips entities that are not configured and never calls bulkIndex when nothing is indexable', async () => {
|
|
177
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
178
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, {
|
|
179
|
+
queryEngine: makeQueryEngine({}),
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
const result = await indexer.indexRecordsById({
|
|
183
|
+
items: [{ entityId: 'unknown:entity', recordId: 'rec-1' }],
|
|
184
|
+
tenantId: 'tenant-123',
|
|
185
|
+
organizationId: null,
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
expect(searchService.bulkIndex).not.toHaveBeenCalled()
|
|
189
|
+
expect(result).toEqual({ indexed: 0, skipped: 1 })
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
it('counts records dropped for a missing id as skipped so the totals add up', async () => {
|
|
193
|
+
const searchService = { bulkIndex: jest.fn().mockResolvedValue(undefined) }
|
|
194
|
+
const queryEngine: QueryEngine = {
|
|
195
|
+
query: jest.fn(async (_entity, opts) => {
|
|
196
|
+
const wantedId = (opts?.filters as { id?: string } | undefined)?.id
|
|
197
|
+
const item = wantedId === 'no-id' ? { name: 'Ghost' } : { id: wantedId, name: 'Alpha' }
|
|
198
|
+
return { items: [item], total: 1 } as QueryResult
|
|
199
|
+
}),
|
|
200
|
+
}
|
|
201
|
+
const indexer = new SearchIndexer(searchService as any, moduleConfigs, { queryEngine })
|
|
202
|
+
|
|
203
|
+
const result = await indexer.indexRecordsById({
|
|
204
|
+
items: [
|
|
205
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
206
|
+
{ entityId: 'test:entity', recordId: 'no-id' },
|
|
207
|
+
],
|
|
208
|
+
tenantId: 'tenant-123',
|
|
209
|
+
organizationId: null,
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
expect(result).toEqual({ indexed: 1, skipped: 1 })
|
|
213
|
+
})
|
|
214
|
+
})
|
|
@@ -431,6 +431,30 @@ describe('SearchService', () => {
|
|
|
431
431
|
expect(strategy.index).toHaveBeenCalledTimes(2)
|
|
432
432
|
})
|
|
433
433
|
|
|
434
|
+
it('should bound in-flight writes when falling back to individual indexing', async () => {
|
|
435
|
+
let inFlight = 0
|
|
436
|
+
let peakInFlight = 0
|
|
437
|
+
const strategy = createMockStrategy({
|
|
438
|
+
id: 'test',
|
|
439
|
+
bulkIndex: undefined,
|
|
440
|
+
index: jest.fn(async () => {
|
|
441
|
+
inFlight++
|
|
442
|
+
peakInFlight = Math.max(peakInFlight, inFlight)
|
|
443
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
|
444
|
+
inFlight--
|
|
445
|
+
}),
|
|
446
|
+
})
|
|
447
|
+
const service = new SearchService({ strategies: [strategy] })
|
|
448
|
+
const records = Array.from({ length: 50 }, (_, index) =>
|
|
449
|
+
createMockRecord({ recordId: `rec-${index}` }),
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
await service.bulkIndex(records)
|
|
453
|
+
|
|
454
|
+
expect(strategy.index).toHaveBeenCalledTimes(50)
|
|
455
|
+
expect(peakInFlight).toBeLessThanOrEqual(4)
|
|
456
|
+
})
|
|
457
|
+
|
|
434
458
|
it('should do nothing when records array is empty', async () => {
|
|
435
459
|
const strategy = createMockStrategy({ id: 'test' })
|
|
436
460
|
const service = new SearchService({ strategies: [strategy] })
|
|
@@ -451,6 +451,7 @@ describe('Fulltext Index Worker', () => {
|
|
|
451
451
|
const mockSearchIndexer = {
|
|
452
452
|
getEntityConfig: jest.fn().mockReturnValue(null),
|
|
453
453
|
indexRecordById: jest.fn().mockResolvedValue({ action: 'indexed', created: true }),
|
|
454
|
+
indexRecordsById: jest.fn().mockResolvedValue({ indexed: 0, skipped: 0 }),
|
|
454
455
|
}
|
|
455
456
|
|
|
456
457
|
const mockEm = {
|
|
@@ -471,6 +472,7 @@ describe('Fulltext Index Worker', () => {
|
|
|
471
472
|
;(hasActiveReindexProgress as jest.Mock).mockResolvedValue(true)
|
|
472
473
|
mockFulltextStrategy.isAvailable.mockResolvedValue(true)
|
|
473
474
|
mockSearchIndexer.indexRecordById.mockResolvedValue({ action: 'indexed', created: true })
|
|
475
|
+
mockSearchIndexer.indexRecordsById.mockResolvedValue({ indexed: 0, skipped: 0 })
|
|
474
476
|
})
|
|
475
477
|
|
|
476
478
|
it('should skip job with missing tenantId', async () => {
|
|
@@ -486,12 +488,13 @@ describe('Fulltext Index Worker', () => {
|
|
|
486
488
|
expect(mockFulltextStrategy.bulkIndex).not.toHaveBeenCalled()
|
|
487
489
|
})
|
|
488
490
|
|
|
489
|
-
it('should index
|
|
491
|
+
it('should index the whole batch in a single indexRecordsById call when jobType is batch-index', async () => {
|
|
490
492
|
// Use minimal record format (just entityId + recordId)
|
|
491
493
|
const records = [
|
|
492
494
|
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
493
495
|
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
494
496
|
]
|
|
497
|
+
mockSearchIndexer.indexRecordsById.mockResolvedValueOnce({ indexed: 2, skipped: 0 })
|
|
495
498
|
const job = createMockJob<FulltextIndexJobPayload>({
|
|
496
499
|
jobType: 'batch-index',
|
|
497
500
|
tenantId: 'tenant-123',
|
|
@@ -501,26 +504,22 @@ describe('Fulltext Index Worker', () => {
|
|
|
501
504
|
|
|
502
505
|
await handleFulltextIndexJob(job, ctx, mockContainer)
|
|
503
506
|
|
|
504
|
-
// Verify
|
|
505
|
-
|
|
506
|
-
expect(mockSearchIndexer.
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
expect(mockSearchIndexer.indexRecordById).toHaveBeenCalledWith({
|
|
513
|
-
entityId: 'test:entity',
|
|
514
|
-
recordId: 'rec-2',
|
|
507
|
+
// Verify the whole batch is written through exactly one indexRecordsById
|
|
508
|
+
// call, not one indexRecordById call per record.
|
|
509
|
+
expect(mockSearchIndexer.indexRecordsById).toHaveBeenCalledTimes(1)
|
|
510
|
+
expect(mockSearchIndexer.indexRecordsById).toHaveBeenCalledWith({
|
|
511
|
+
items: [
|
|
512
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
513
|
+
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
514
|
+
],
|
|
515
515
|
tenantId: 'tenant-123',
|
|
516
516
|
organizationId: undefined,
|
|
517
517
|
})
|
|
518
|
+
expect(mockSearchIndexer.indexRecordById).not.toHaveBeenCalled()
|
|
518
519
|
})
|
|
519
520
|
|
|
520
521
|
it('counts handled fulltext batch records as processed so progress can complete', async () => {
|
|
521
|
-
mockSearchIndexer.
|
|
522
|
-
.mockResolvedValueOnce({ action: 'skipped' })
|
|
523
|
-
.mockResolvedValueOnce({ action: 'skipped' })
|
|
522
|
+
mockSearchIndexer.indexRecordsById.mockResolvedValueOnce({ indexed: 0, skipped: 2 })
|
|
524
523
|
const records = [
|
|
525
524
|
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
526
525
|
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
@@ -543,7 +542,7 @@ describe('Fulltext Index Worker', () => {
|
|
|
543
542
|
|
|
544
543
|
await handleFulltextIndexJob(job, createMockJobContext(), containerWithProgress)
|
|
545
544
|
|
|
546
|
-
expect(mockSearchIndexer.
|
|
545
|
+
expect(mockSearchIndexer.indexRecordsById).toHaveBeenCalledTimes(1)
|
|
547
546
|
expect(updateReindexProgress).toHaveBeenCalledWith(mockDb, 'tenant-123', 'fulltext', 2, 'org-456')
|
|
548
547
|
expect(incrementReindexProgress).toHaveBeenCalledWith(
|
|
549
548
|
expect.objectContaining({ type: 'fulltext', tenantId: 'tenant-123', delta: 2 }),
|
|
@@ -551,6 +550,36 @@ describe('Fulltext Index Worker', () => {
|
|
|
551
550
|
expect(clearReindexLock).toHaveBeenCalledWith(mockDb, 'tenant-123', 'fulltext', 'org-456')
|
|
552
551
|
})
|
|
553
552
|
|
|
553
|
+
it('re-throws a failed fulltext batch write without advancing reindex progress so the queue retries it', async () => {
|
|
554
|
+
mockSearchIndexer.indexRecordsById.mockRejectedValueOnce(new Error('meilisearch unavailable'))
|
|
555
|
+
const records = [
|
|
556
|
+
{ entityId: 'test:entity', recordId: 'rec-1' },
|
|
557
|
+
{ entityId: 'test:entity', recordId: 'rec-2' },
|
|
558
|
+
]
|
|
559
|
+
const containerWithProgress: HandlerContext = {
|
|
560
|
+
resolve: jest.fn((name: string) => {
|
|
561
|
+
if (name === 'searchStrategies') return [mockFulltextStrategy]
|
|
562
|
+
if (name === 'em') return mockEm
|
|
563
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
564
|
+
if (name === 'progressService') return { id: 'progress' }
|
|
565
|
+
throw new Error(`Unknown service: ${name}`)
|
|
566
|
+
}) as HandlerContext['resolve'],
|
|
567
|
+
}
|
|
568
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
569
|
+
jobType: 'batch-index',
|
|
570
|
+
tenantId: 'tenant-123',
|
|
571
|
+
organizationId: 'org-456',
|
|
572
|
+
records,
|
|
573
|
+
})
|
|
574
|
+
|
|
575
|
+
await expect(
|
|
576
|
+
handleFulltextIndexJob(job, createMockJobContext(), containerWithProgress),
|
|
577
|
+
).rejects.toThrow('meilisearch unavailable')
|
|
578
|
+
|
|
579
|
+
expect(updateReindexProgress).not.toHaveBeenCalled()
|
|
580
|
+
expect(incrementReindexProgress).not.toHaveBeenCalled()
|
|
581
|
+
})
|
|
582
|
+
|
|
554
583
|
it('clears an orphaned fulltext reindex lock instead of recreating it when no progress job is active', async () => {
|
|
555
584
|
;(hasActiveReindexProgress as jest.Mock).mockResolvedValueOnce(false)
|
|
556
585
|
const records = [{ entityId: 'test:entity', recordId: 'rec-1' }]
|
|
@@ -572,7 +601,7 @@ describe('Fulltext Index Worker', () => {
|
|
|
572
601
|
|
|
573
602
|
await handleFulltextIndexJob(job, createMockJobContext(), containerWithProgress)
|
|
574
603
|
|
|
575
|
-
expect(mockSearchIndexer.
|
|
604
|
+
expect(mockSearchIndexer.indexRecordsById).toHaveBeenCalledTimes(1)
|
|
576
605
|
expect(updateReindexProgress).not.toHaveBeenCalled()
|
|
577
606
|
expect(incrementReindexProgress).not.toHaveBeenCalled()
|
|
578
607
|
expect(clearReindexLock).toHaveBeenCalledWith(mockDb, 'tenant-123', 'fulltext', 'org-456')
|
|
@@ -33,6 +33,15 @@ export type IndexRecordParams = {
|
|
|
33
33
|
customFields?: Record<string, unknown>
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Parameters for indexing a batch of records by id in a single bulk write.
|
|
38
|
+
*/
|
|
39
|
+
export type IndexRecordsByIdParams = {
|
|
40
|
+
items: Array<{ entityId: EntityId; recordId: string }>
|
|
41
|
+
tenantId: string
|
|
42
|
+
organizationId?: string | null
|
|
43
|
+
}
|
|
44
|
+
|
|
36
45
|
/**
|
|
37
46
|
* Parameters for deleting a record from the search index.
|
|
38
47
|
*/
|
|
@@ -342,6 +351,81 @@ export class SearchIndexer {
|
|
|
342
351
|
}
|
|
343
352
|
}
|
|
344
353
|
|
|
354
|
+
/**
|
|
355
|
+
* Index a batch of records by id in a single bulk write.
|
|
356
|
+
* Unlike calling indexRecordById() in a loop, this loads each record fresh
|
|
357
|
+
* (same as indexRecordById) but flushes the whole batch through a single
|
|
358
|
+
* searchService.bulkIndex() call. Strategies that implement bulkIndex then
|
|
359
|
+
* collapse the batch into one write; strategies without it still write per
|
|
360
|
+
* record, at the bounded concurrency SearchService applies.
|
|
361
|
+
*/
|
|
362
|
+
async indexRecordsById(params: IndexRecordsByIdParams): Promise<{ indexed: number; skipped: number }> {
|
|
363
|
+
const { items, tenantId, organizationId } = params
|
|
364
|
+
if (!this.queryEngine || items.length === 0) {
|
|
365
|
+
return { indexed: 0, skipped: items.length }
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
const recordIdsByEntity = new Map<EntityId, string[]>()
|
|
369
|
+
for (const item of items) {
|
|
370
|
+
const list = recordIdsByEntity.get(item.entityId) ?? []
|
|
371
|
+
list.push(item.recordId)
|
|
372
|
+
recordIdsByEntity.set(item.entityId, list)
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const allRecords: IndexableRecord[] = []
|
|
376
|
+
let skipped = 0
|
|
377
|
+
|
|
378
|
+
for (const [entityId, recordIds] of recordIdsByEntity) {
|
|
379
|
+
const config = this.entityConfigMap.get(entityId)
|
|
380
|
+
if (!config || config.enabled === false) {
|
|
381
|
+
skipped += recordIds.length
|
|
382
|
+
continue
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
for (const recordId of recordIds) {
|
|
386
|
+
try {
|
|
387
|
+
const result = await this.queryEngine.query(entityId, {
|
|
388
|
+
tenantId,
|
|
389
|
+
organizationId: organizationId ?? undefined,
|
|
390
|
+
filters: { id: recordId },
|
|
391
|
+
includeCustomFields: true,
|
|
392
|
+
page: { page: 1, pageSize: 1 },
|
|
393
|
+
skipAutoReindex: true,
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
const record = result.items[0] as Record<string, unknown> | undefined
|
|
397
|
+
if (!record) {
|
|
398
|
+
skipped++
|
|
399
|
+
continue
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const { records: built, dropped } = await this.buildIndexableRecords(
|
|
403
|
+
entityId,
|
|
404
|
+
tenantId,
|
|
405
|
+
organizationId ?? null,
|
|
406
|
+
[record],
|
|
407
|
+
config,
|
|
408
|
+
)
|
|
409
|
+
skipped += dropped
|
|
410
|
+
allRecords.push(...built)
|
|
411
|
+
} catch (error) {
|
|
412
|
+
skipped++
|
|
413
|
+
searchError('SearchIndexer', 'Failed to load record for batch indexing', {
|
|
414
|
+
entityId,
|
|
415
|
+
recordId,
|
|
416
|
+
error: error instanceof Error ? error.message : error,
|
|
417
|
+
})
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
if (allRecords.length > 0) {
|
|
423
|
+
await this.searchService.bulkIndex(allRecords)
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
return { indexed: allRecords.length, skipped }
|
|
427
|
+
}
|
|
428
|
+
|
|
345
429
|
/**
|
|
346
430
|
* Delete a record from the search index.
|
|
347
431
|
*/
|