@open-mercato/search 0.10.1-develop.8227.1.33f80330f2 → 0.10.1-develop.8229.1.61fb2a0f4c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -7
- package/dist/di.js +16 -5
- package/dist/di.js.map +3 -3
- package/dist/indexer/search-indexer.js +136 -36
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +7 -3
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +14 -6
- package/dist/lib/encrypted-custom-field-resolver.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +141 -27
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/debug.test.ts +42 -1
- package/src/__tests__/di-encrypted-custom-fields.test.ts +113 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +76 -28
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +2 -2
- package/src/__tests__/search-indexer-encrypted-custom-fields.boundary.test.ts +214 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.customers.test.ts +146 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.test.ts +425 -0
- package/src/__tests__/vector-index-security.test.ts +249 -3
- package/src/di.ts +19 -5
- package/src/indexer/search-indexer.ts +188 -36
- package/src/lib/debug.ts +16 -3
- package/src/lib/encrypted-custom-field-resolver.ts +35 -16
- package/src/vector/services/vector-index.service.ts +178 -23
package/AGENTS.md
CHANGED
|
@@ -4,13 +4,14 @@ When working on search functionality, use this guide. It covers indexing, queryi
|
|
|
4
4
|
|
|
5
5
|
## Always
|
|
6
6
|
|
|
7
|
-
1. **MUST**
|
|
8
|
-
2. **MUST**
|
|
9
|
-
3. **MUST** define `
|
|
10
|
-
4. **MUST**
|
|
11
|
-
5. **MUST**
|
|
12
|
-
6. **MUST** use `
|
|
13
|
-
7. **MUST**
|
|
7
|
+
1. **MUST** resolve `SearchIndexer` from DI; direct `SearchIndexer` or `VectorIndexService` constructors pass `encryptedCustomFieldKeysResolver` and fail closed without it.
|
|
8
|
+
2. **MUST** create a `search.ts` file for every module with searchable entities.
|
|
9
|
+
3. **MUST** define `fieldPolicy.excluded` for any sensitive fields (passwords, tokens, SSNs, bank accounts) -- never allow them into any index.
|
|
10
|
+
4. **MUST** define `formatResult` for every entity that uses the tokens strategy -- without it, users see raw UUIDs instead of names.
|
|
11
|
+
5. **MUST** include `checksumSource` in every `buildSource` return value so the indexer can detect changes and skip redundant re-embedding.
|
|
12
|
+
6. **MUST** use the `entityId` format `module:entity_name` and ensure it matches the entity registry exactly.
|
|
13
|
+
7. **MUST** use `fieldPolicy.hashOnly` for PII fields (email, phone, tax_id) that need exact-match filtering but not fuzzy search.
|
|
14
|
+
8. **MUST** declare `aclFeatures` on every entity, naming the owning module's view feature(s) — the same gate that entity's own list/read route enforces. `search.global` only authorizes *using* search; `aclFeatures` is what authorizes *reading the records*. Global search, the hybrid `GET /api/search/search` endpoint and the AI tools all fail closed, so an entity without it silently vanishes from results for every non-superadmin.
|
|
14
15
|
|
|
15
16
|
## Ask First
|
|
16
17
|
|
package/dist/di.js
CHANGED
|
@@ -8,9 +8,14 @@ import { createFulltextDriver } from "./fulltext/drivers/index.js";
|
|
|
8
8
|
import { SearchIndexer } from "./indexer/search-indexer.js";
|
|
9
9
|
import { createPresenterEnricher } from "./lib/presenter-enricher.js";
|
|
10
10
|
import { createEncryptionMapResolver } from "./lib/encryption-map-resolver.js";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
createEncryptedCustomFieldKeysResolver
|
|
13
|
+
} from "./lib/encrypted-custom-field-resolver.js";
|
|
12
14
|
import { reportSearchError } from "./lib/debug.js";
|
|
13
15
|
import { createEncryptionMapResolver as createEncryptionMapResolver2 } from "./lib/encryption-map-resolver.js";
|
|
16
|
+
import {
|
|
17
|
+
createEncryptedCustomFieldKeysResolver as createEncryptedCustomFieldKeysResolver2
|
|
18
|
+
} from "./lib/encrypted-custom-field-resolver.js";
|
|
14
19
|
const FULLTEXT_DRIVER_KEY = "__omSearchFulltextDriver__";
|
|
15
20
|
function getSearchModuleGlobals() {
|
|
16
21
|
return globalThis;
|
|
@@ -19,6 +24,11 @@ function shouldExcludeEncryptedFields() {
|
|
|
19
24
|
const raw = (process.env.SEARCH_EXCLUDE_ENCRYPTED_FIELDS ?? "").toLowerCase();
|
|
20
25
|
return raw === "1" || raw === "true" || raw === "yes" || raw === "on";
|
|
21
26
|
}
|
|
27
|
+
function createUnavailableEncryptedCustomFieldKeysResolver() {
|
|
28
|
+
return async () => {
|
|
29
|
+
throw new Error("[internal] Custom-field definitions are unavailable; refusing to index potentially encrypted custom fields");
|
|
30
|
+
};
|
|
31
|
+
}
|
|
22
32
|
function createUnavailableEncryptionMapResolver() {
|
|
23
33
|
return async () => {
|
|
24
34
|
throw new Error("[internal] Encryption map lookup is unavailable; refusing to index potentially encrypted fields");
|
|
@@ -138,12 +148,12 @@ function registerSearchModule(container, options) {
|
|
|
138
148
|
let encryptedCustomFieldKeysResolver;
|
|
139
149
|
try {
|
|
140
150
|
const em = container.resolve("em");
|
|
141
|
-
encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(
|
|
151
|
+
encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(
|
|
152
|
+
em.getKysely()
|
|
153
|
+
);
|
|
142
154
|
} catch (error) {
|
|
143
155
|
reportSearchError("SearchModule", error, "search.custom_field_definitions_unavailable");
|
|
144
|
-
encryptedCustomFieldKeysResolver =
|
|
145
|
-
throw new Error("[internal] Custom-field definitions are unavailable");
|
|
146
|
-
};
|
|
156
|
+
encryptedCustomFieldKeysResolver = createUnavailableEncryptedCustomFieldKeysResolver();
|
|
147
157
|
}
|
|
148
158
|
const searchIndexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
149
159
|
queryEngine,
|
|
@@ -173,6 +183,7 @@ function addSearchStrategy(container, strategy) {
|
|
|
173
183
|
}
|
|
174
184
|
export {
|
|
175
185
|
addSearchStrategy,
|
|
186
|
+
createEncryptedCustomFieldKeysResolver2 as createEncryptedCustomFieldKeysResolver,
|
|
176
187
|
createEncryptionMapResolver2 as createEncryptionMapResolver,
|
|
177
188
|
registerSearchModule
|
|
178
189
|
};
|
package/dist/di.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/di.ts"],
|
|
4
|
-
"sourcesContent": ["import { resolveEntityIndexStrategies } from './lib/entity-strategy-policy'\nimport { asValue } from 'awilix'\nimport type { Kysely } from 'kysely'\nimport type { FullTextSearchDriver } from './fulltext/types'\nimport { SearchService } from './service'\nimport { TokenSearchStrategy } from './strategies/token.strategy'\nimport { VectorSearchStrategy, type EmbeddingService } from './strategies/vector.strategy'\nimport { FullTextSearchStrategy } from './strategies/fulltext.strategy'\nimport { createFulltextDriver } from './fulltext/drivers'\nimport { SearchIndexer } from './indexer/search-indexer'\nimport type {\n SearchStrategy,\n ResultMergeConfig,\n SearchModuleConfig,\n SearchFieldPolicy,\n SearchEntityConfig,\n PresenterEnricherFn,\n} from './types'\nimport type { VectorDriver } from './vector/types'\nimport type { QueryEngine } from '@open-mercato/shared/lib/query/types'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport type { SearchStrategyId } from '@open-mercato/shared/modules/search'\nimport type { Queue } from '@open-mercato/queue'\nimport type { FulltextIndexJobPayload } from './queue/fulltext-indexing'\nimport type { VectorIndexJobPayload } from './queue/vector-indexing'\nimport type { EncryptionMapEntry } from './lib/field-policy'\nimport type { TenantDataEncryptionService } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'\nimport { createPresenterEnricher } from './lib/presenter-enricher'\nimport { createEncryptionMapResolver } from './lib/encryption-map-resolver'\nimport { createEncryptedCustomFieldKeysResolver, type CustomFieldDefinitionsDatabase, type EncryptedCustomFieldKeysResolver } from './lib/encrypted-custom-field-resolver'\nimport { reportSearchError } from './lib/debug'\n\nexport { createEncryptionMapResolver } from './lib/encryption-map-resolver'\n\ntype KyselyEntityManager = {\n getKysely<Database>(): Kysely<Database>\n}\n\nconst FULLTEXT_DRIVER_KEY = '__omSearchFulltextDriver__'\n\ntype SearchModuleGlobals = {\n [FULLTEXT_DRIVER_KEY]?: FullTextSearchDriver\n}\n\nfunction getSearchModuleGlobals(): SearchModuleGlobals {\n return globalThis as unknown as SearchModuleGlobals\n}\n\n/**\n * Check if encrypted fields should be excluded from search indexing.\n * Controlled by SEARCH_EXCLUDE_ENCRYPTED_FIELDS environment variable.\n * Default: false (index all fields including decrypted data)\n */\nfunction shouldExcludeEncryptedFields(): boolean {\n const raw = (process.env.SEARCH_EXCLUDE_ENCRYPTED_FIELDS ?? '').toLowerCase()\n return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on'\n}\n\nfunction createUnavailableEncryptionMapResolver(): (entityId: EntityId) => Promise<EncryptionMapEntry[]> {\n return async () => {\n throw new Error('[internal] Encryption map lookup is unavailable; refusing to index potentially encrypted fields')\n }\n}\n\n/**\n * Container interface - minimal subset needed for registration.\n */\nexport interface SearchContainer {\n resolve<T = unknown>(name: string): T\n register(registrations: Record<string, unknown>): void\n}\n\n/**\n * Configuration options for search module registration.\n */\nexport type SearchModuleOptions = {\n /** Override default strategies to use */\n defaultStrategies?: SearchStrategyId[]\n /** Override merge configuration */\n mergeConfig?: ResultMergeConfig\n /** Skip token strategy registration */\n skipTokens?: boolean\n /** Skip vector strategy registration */\n skipVector?: boolean\n /** Skip fulltext strategy registration */\n skipFulltext?: boolean\n /** Module configurations (from generated/search.generated.ts) */\n moduleConfigs?: SearchModuleConfig[]\n}\n\n/**\n * Register the search module in the DI container.\n *\n * This creates and registers:\n * - SearchService instance\n * - All configured search strategies\n *\n * @param container - Awilix container\n * @param options - Optional configuration overrides\n */\nexport function registerSearchModule(\n container: SearchContainer,\n options?: SearchModuleOptions,\n): void {\n const moduleConfigs = (options?.moduleConfigs ?? []).map((moduleConfig) => ({\n ...moduleConfig,\n entities: moduleConfig.entities.map((entityConfig) => {\n const buildSource = entityConfig.buildSource\n return buildSource\n ? { ...entityConfig, buildSource: (ctx: Parameters<typeof buildSource>[0]) => buildSource.call(entityConfig, { ...ctx, container }) }\n : entityConfig\n }),\n }))\n const strategies: SearchStrategy[] = []\n\n // Token strategy (always available unless explicitly skipped)\n if (!options?.skipTokens) {\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n strategies.push(new TokenSearchStrategy(db))\n } catch {\n // Kysely not available via em, skipping TokenSearchStrategy\n }\n }\n\n // Vector strategy (requires embedding service and driver)\n // Note: We register even if not currently available - availability is checked at search time\n // via isAvailable(). The embedding config may be loaded later from the database.\n if (!options?.skipVector) {\n try {\n const embeddingService = container.resolve<EmbeddingService>('vectorEmbeddingService')\n const drivers = container.resolve<VectorDriver[]>('vectorDrivers')\n const primaryDriver = drivers?.[0]\n\n if (embeddingService && primaryDriver) {\n strategies.push(new VectorSearchStrategy(embeddingService, primaryDriver))\n }\n } catch {\n // Vector module not available, skipping VectorSearchStrategy\n }\n }\n\n // Build entity config map for field policy resolution\n const entityConfigMap = new Map<EntityId, SearchEntityConfig>()\n for (const moduleConfig of moduleConfigs) {\n for (const entityConfig of moduleConfig.entities) {\n if (entityConfig.enabled !== false) {\n entityConfigMap.set(entityConfig.entityId as EntityId, entityConfig)\n }\n }\n }\n\n // Fulltext strategy (requires driver configuration, e.g., MEILISEARCH_HOST)\n if (!options?.skipFulltext) {\n const excludeEncrypted = shouldExcludeEncryptedFields()\n const singletonCacheEnabled = process.env.SEARCH_DISABLE_SINGLETON_CACHE !== '1'\n const g = getSearchModuleGlobals()\n\n // The fulltext driver is safe to memoize only when SEARCH_EXCLUDE_ENCRYPTED_FIELDS is\n // OFF \u2014 in that case the driver holds no request-scoped Kysely reference. When the flag\n // is ON the encryptionMapResolver captures a per-request db handle and must stay\n // per-request to avoid cross-request data leaks.\n const canMemoize = singletonCacheEnabled && !excludeEncrypted\n\n let fulltextDriver: FullTextSearchDriver | null = canMemoize\n ? (g[FULLTEXT_DRIVER_KEY] ?? null)\n : null\n\n if (!fulltextDriver) {\n let encryptionMapResolver: ((entityId: EntityId) => Promise<EncryptionMapEntry[]>) | undefined\n if (excludeEncrypted) {\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n encryptionMapResolver = createEncryptionMapResolver(db)\n } catch {\n encryptionMapResolver = createUnavailableEncryptionMapResolver()\n }\n }\n\n fulltextDriver = createFulltextDriver({\n fieldPolicyResolver: (entityId: EntityId): SearchFieldPolicy | undefined => {\n const config = entityConfigMap.get(entityId)\n return config?.fieldPolicy\n },\n encryptionMapResolver,\n })\n\n if (canMemoize && fulltextDriver) {\n g[FULLTEXT_DRIVER_KEY] = fulltextDriver\n }\n }\n\n if (fulltextDriver) {\n strategies.push(new FullTextSearchStrategy(fulltextDriver))\n }\n }\n\n // Determine default strategies based on what's available\n const defaultStrategies = options?.defaultStrategies ?? determineDefaultStrategies(strategies)\n\n // Try to resolve queryEngine for reindex support and presenter enrichment\n let queryEngine: QueryEngine | undefined\n try {\n queryEngine = container.resolve<QueryEngine>('queryEngine')\n } catch {\n // QueryEngine not available, reindex will be disabled\n }\n\n // Resolve encryption service for decrypting presenter data\n let encryptionService: TenantDataEncryptionService | null = null\n try {\n encryptionService = container.resolve<TenantDataEncryptionService>('tenantEncryptionService')\n } catch {\n // Encryption service not available, presenters won't be decrypted\n }\n\n // Create presenter enricher for database-based presenter resolution\n let presenterEnricher: PresenterEnricherFn | undefined\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n presenterEnricher = createPresenterEnricher(db, entityConfigMap, queryEngine, encryptionService)\n } catch {\n // Kysely not available, presenter enrichment disabled\n }\n\n // Create search service\n const searchService = new SearchService({\n strategies,\n resolveEntityStrategies: (entityId) => resolveEntityIndexStrategies(entityId, moduleConfigs),\n defaultStrategies,\n fallbackStrategy: 'tokens',\n mergeConfig: options?.mergeConfig ?? {\n duplicateHandling: 'highest_score',\n strategyWeights: {\n fulltext: 1.2,\n vector: 1.0,\n tokens: 0.8,\n },\n },\n presenterEnricher,\n })\n\n // Try to resolve fulltextIndexQueue for queue-based reindexing\n let fulltextQueue: Queue<FulltextIndexJobPayload> | undefined\n try {\n fulltextQueue = container.resolve<Queue<FulltextIndexJobPayload>>('fulltextIndexQueue')\n } catch {\n // Queue not available, queue-based fulltext reindex will be disabled\n }\n\n // Try to resolve vectorIndexQueue for queue-based vector reindexing\n let vectorQueue: Queue<VectorIndexJobPayload> | undefined\n try {\n vectorQueue = container.resolve<Queue<VectorIndexJobPayload>>('vectorIndexQueue')\n } catch {\n // Queue not available, queue-based vector reindex will be disabled\n }\n\n let encryptedCustomFieldKeysResolver: EncryptedCustomFieldKeysResolver\n try {\n const em = container.resolve<KyselyEntityManager>('em')\n encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(em.getKysely<CustomFieldDefinitionsDatabase>())\n } catch (error) {\n reportSearchError('SearchModule', error, 'search.custom_field_definitions_unavailable')\n encryptedCustomFieldKeysResolver = async () => {\n throw new Error('[internal] Custom-field definitions are unavailable')\n }\n }\n\n const searchIndexer = new SearchIndexer(searchService, moduleConfigs, {\n queryEngine,\n fulltextQueue,\n vectorQueue,\n encryptedCustomFieldKeysResolver,\n })\n\n // Register in container\n container.register({\n searchService: asValue(searchService),\n searchStrategies: asValue(strategies),\n searchIndexer: asValue(searchIndexer),\n })\n}\n\n/**\n * Determine default strategy order based on available strategies.\n * Prefers fulltext > vector > tokens.\n */\nfunction determineDefaultStrategies(strategies: SearchStrategy[]): SearchStrategyId[] {\n const available = new Set(strategies.map((s) => s.id))\n const defaults: SearchStrategyId[] = []\n\n if (available.has('fulltext')) defaults.push('fulltext')\n if (available.has('vector')) defaults.push('vector')\n if (available.has('tokens')) defaults.push('tokens')\n\n return defaults.length > 0 ? defaults : ['tokens']\n}\n\n/**\n * Helper to add a custom strategy to an existing SearchService.\n *\n * @param container - DI container\n * @param strategy - Strategy to add\n */\nexport function addSearchStrategy(container: SearchContainer, strategy: SearchStrategy): void {\n const service = container.resolve<SearchService>('searchService')\n service.registerStrategy(strategy)\n\n const strategies = container.resolve<SearchStrategy[]>('searchStrategies')\n strategies.push(strategy)\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,oCAAoC;AAC7C,SAAS,eAAe;AAGxB,SAAS,qBAAqB;AAC9B,SAAS,2BAA2B;AACpC,SAAS,4BAAmD;AAC5D,SAAS,8BAA8B;AACvC,SAAS,4BAA4B;AACrC,SAAS,qBAAqB;AAkB9B,SAAS,+BAA+B;AACxC,SAAS,mCAAmC;AAC5C,
|
|
6
|
-
"names": ["createEncryptionMapResolver"]
|
|
4
|
+
"sourcesContent": ["import { resolveEntityIndexStrategies } from './lib/entity-strategy-policy'\nimport { asValue } from 'awilix'\nimport type { Kysely } from 'kysely'\nimport type { FullTextSearchDriver } from './fulltext/types'\nimport { SearchService } from './service'\nimport { TokenSearchStrategy } from './strategies/token.strategy'\nimport { VectorSearchStrategy, type EmbeddingService } from './strategies/vector.strategy'\nimport { FullTextSearchStrategy } from './strategies/fulltext.strategy'\nimport { createFulltextDriver } from './fulltext/drivers'\nimport { SearchIndexer } from './indexer/search-indexer'\nimport type {\n SearchStrategy,\n ResultMergeConfig,\n SearchModuleConfig,\n SearchFieldPolicy,\n SearchEntityConfig,\n PresenterEnricherFn,\n} from './types'\nimport type { VectorDriver } from './vector/types'\nimport type { QueryEngine } from '@open-mercato/shared/lib/query/types'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport type { SearchStrategyId } from '@open-mercato/shared/modules/search'\nimport type { Queue } from '@open-mercato/queue'\nimport type { FulltextIndexJobPayload } from './queue/fulltext-indexing'\nimport type { VectorIndexJobPayload } from './queue/vector-indexing'\nimport type { EncryptionMapEntry } from './lib/field-policy'\nimport type { TenantDataEncryptionService } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'\nimport { createPresenterEnricher } from './lib/presenter-enricher'\nimport { createEncryptionMapResolver } from './lib/encryption-map-resolver'\nimport {\n createEncryptedCustomFieldKeysResolver,\n type CustomFieldDefinitionsDatabase,\n type EncryptedCustomFieldKeysResolver,\n} from './lib/encrypted-custom-field-resolver'\nimport { reportSearchError } from './lib/debug'\n\nexport { createEncryptionMapResolver } from './lib/encryption-map-resolver'\nexport {\n createEncryptedCustomFieldKeysResolver,\n type EncryptedCustomFieldKeysResolver,\n} from './lib/encrypted-custom-field-resolver'\n\ntype KyselyEntityManager = {\n getKysely<Database>(): Kysely<Database>\n}\n\nconst FULLTEXT_DRIVER_KEY = '__omSearchFulltextDriver__'\n\ntype SearchModuleGlobals = {\n [FULLTEXT_DRIVER_KEY]?: FullTextSearchDriver\n}\n\nfunction getSearchModuleGlobals(): SearchModuleGlobals {\n return globalThis as unknown as SearchModuleGlobals\n}\n\n/**\n * Check if encrypted fields should be excluded from search indexing.\n * Controlled by SEARCH_EXCLUDE_ENCRYPTED_FIELDS environment variable.\n * Default: false (index all fields including decrypted data)\n */\nfunction shouldExcludeEncryptedFields(): boolean {\n const raw = (process.env.SEARCH_EXCLUDE_ENCRYPTED_FIELDS ?? '').toLowerCase()\n return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on'\n}\n\nfunction createUnavailableEncryptedCustomFieldKeysResolver(): EncryptedCustomFieldKeysResolver {\n return async () => {\n throw new Error('[internal] Custom-field definitions are unavailable; refusing to index potentially encrypted custom fields')\n }\n}\n\nfunction createUnavailableEncryptionMapResolver(): (entityId: EntityId) => Promise<EncryptionMapEntry[]> {\n return async () => {\n throw new Error('[internal] Encryption map lookup is unavailable; refusing to index potentially encrypted fields')\n }\n}\n\n/**\n * Container interface - minimal subset needed for registration.\n */\nexport interface SearchContainer {\n resolve<T = unknown>(name: string): T\n register(registrations: Record<string, unknown>): void\n}\n\n/**\n * Configuration options for search module registration.\n */\nexport type SearchModuleOptions = {\n /** Override default strategies to use */\n defaultStrategies?: SearchStrategyId[]\n /** Override merge configuration */\n mergeConfig?: ResultMergeConfig\n /** Skip token strategy registration */\n skipTokens?: boolean\n /** Skip vector strategy registration */\n skipVector?: boolean\n /** Skip fulltext strategy registration */\n skipFulltext?: boolean\n /** Module configurations (from generated/search.generated.ts) */\n moduleConfigs?: SearchModuleConfig[]\n}\n\n/**\n * Register the search module in the DI container.\n *\n * This creates and registers:\n * - SearchService instance\n * - All configured search strategies\n *\n * @param container - Awilix container\n * @param options - Optional configuration overrides\n */\nexport function registerSearchModule(\n container: SearchContainer,\n options?: SearchModuleOptions,\n): void {\n const moduleConfigs = (options?.moduleConfigs ?? []).map((moduleConfig) => ({\n ...moduleConfig,\n entities: moduleConfig.entities.map((entityConfig) => {\n const buildSource = entityConfig.buildSource\n return buildSource\n ? { ...entityConfig, buildSource: (ctx: Parameters<typeof buildSource>[0]) => buildSource.call(entityConfig, { ...ctx, container }) }\n : entityConfig\n }),\n }))\n const strategies: SearchStrategy[] = []\n\n // Token strategy (always available unless explicitly skipped)\n if (!options?.skipTokens) {\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n strategies.push(new TokenSearchStrategy(db))\n } catch {\n // Kysely not available via em, skipping TokenSearchStrategy\n }\n }\n\n // Vector strategy (requires embedding service and driver)\n // Note: We register even if not currently available - availability is checked at search time\n // via isAvailable(). The embedding config may be loaded later from the database.\n if (!options?.skipVector) {\n try {\n const embeddingService = container.resolve<EmbeddingService>('vectorEmbeddingService')\n const drivers = container.resolve<VectorDriver[]>('vectorDrivers')\n const primaryDriver = drivers?.[0]\n\n if (embeddingService && primaryDriver) {\n strategies.push(new VectorSearchStrategy(embeddingService, primaryDriver))\n }\n } catch {\n // Vector module not available, skipping VectorSearchStrategy\n }\n }\n\n // Build entity config map for field policy resolution\n const entityConfigMap = new Map<EntityId, SearchEntityConfig>()\n for (const moduleConfig of moduleConfigs) {\n for (const entityConfig of moduleConfig.entities) {\n if (entityConfig.enabled !== false) {\n entityConfigMap.set(entityConfig.entityId as EntityId, entityConfig)\n }\n }\n }\n\n // Fulltext strategy (requires driver configuration, e.g., MEILISEARCH_HOST)\n if (!options?.skipFulltext) {\n const excludeEncrypted = shouldExcludeEncryptedFields()\n const singletonCacheEnabled = process.env.SEARCH_DISABLE_SINGLETON_CACHE !== '1'\n const g = getSearchModuleGlobals()\n\n // The fulltext driver is safe to memoize only when SEARCH_EXCLUDE_ENCRYPTED_FIELDS is\n // OFF \u2014 in that case the driver holds no request-scoped Kysely reference. When the flag\n // is ON the encryptionMapResolver captures a per-request db handle and must stay\n // per-request to avoid cross-request data leaks.\n const canMemoize = singletonCacheEnabled && !excludeEncrypted\n\n let fulltextDriver: FullTextSearchDriver | null = canMemoize\n ? (g[FULLTEXT_DRIVER_KEY] ?? null)\n : null\n\n if (!fulltextDriver) {\n let encryptionMapResolver: ((entityId: EntityId) => Promise<EncryptionMapEntry[]>) | undefined\n if (excludeEncrypted) {\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n encryptionMapResolver = createEncryptionMapResolver(db)\n } catch {\n encryptionMapResolver = createUnavailableEncryptionMapResolver()\n }\n }\n\n fulltextDriver = createFulltextDriver({\n fieldPolicyResolver: (entityId: EntityId): SearchFieldPolicy | undefined => {\n const config = entityConfigMap.get(entityId)\n return config?.fieldPolicy\n },\n encryptionMapResolver,\n })\n\n if (canMemoize && fulltextDriver) {\n g[FULLTEXT_DRIVER_KEY] = fulltextDriver\n }\n }\n\n if (fulltextDriver) {\n strategies.push(new FullTextSearchStrategy(fulltextDriver))\n }\n }\n\n // Determine default strategies based on what's available\n const defaultStrategies = options?.defaultStrategies ?? determineDefaultStrategies(strategies)\n\n // Try to resolve queryEngine for reindex support and presenter enrichment\n let queryEngine: QueryEngine | undefined\n try {\n queryEngine = container.resolve<QueryEngine>('queryEngine')\n } catch {\n // QueryEngine not available, reindex will be disabled\n }\n\n // Resolve encryption service for decrypting presenter data\n let encryptionService: TenantDataEncryptionService | null = null\n try {\n encryptionService = container.resolve<TenantDataEncryptionService>('tenantEncryptionService')\n } catch {\n // Encryption service not available, presenters won't be decrypted\n }\n\n // Create presenter enricher for database-based presenter resolution\n let presenterEnricher: PresenterEnricherFn | undefined\n try {\n const em = container.resolve<any>('em')\n const db = em.getKysely() as Kysely<any>\n presenterEnricher = createPresenterEnricher(db, entityConfigMap, queryEngine, encryptionService)\n } catch {\n // Kysely not available, presenter enrichment disabled\n }\n\n // Create search service\n const searchService = new SearchService({\n strategies,\n resolveEntityStrategies: (entityId) => resolveEntityIndexStrategies(entityId, moduleConfigs),\n defaultStrategies,\n fallbackStrategy: 'tokens',\n mergeConfig: options?.mergeConfig ?? {\n duplicateHandling: 'highest_score',\n strategyWeights: {\n fulltext: 1.2,\n vector: 1.0,\n tokens: 0.8,\n },\n },\n presenterEnricher,\n })\n\n // Try to resolve fulltextIndexQueue for queue-based reindexing\n let fulltextQueue: Queue<FulltextIndexJobPayload> | undefined\n try {\n fulltextQueue = container.resolve<Queue<FulltextIndexJobPayload>>('fulltextIndexQueue')\n } catch {\n // Queue not available, queue-based fulltext reindex will be disabled\n }\n\n // Try to resolve vectorIndexQueue for queue-based vector reindexing\n let vectorQueue: Queue<VectorIndexJobPayload> | undefined\n try {\n vectorQueue = container.resolve<Queue<VectorIndexJobPayload>>('vectorIndexQueue')\n } catch {\n // Queue not available, queue-based vector reindex will be disabled\n }\n\n let encryptedCustomFieldKeysResolver: EncryptedCustomFieldKeysResolver\n try {\n const em = container.resolve<KyselyEntityManager>('em')\n encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(\n em.getKysely<CustomFieldDefinitionsDatabase>(),\n )\n } catch (error) {\n reportSearchError('SearchModule', error, 'search.custom_field_definitions_unavailable')\n encryptedCustomFieldKeysResolver = createUnavailableEncryptedCustomFieldKeysResolver()\n }\n\n const searchIndexer = new SearchIndexer(searchService, moduleConfigs, {\n queryEngine,\n fulltextQueue,\n vectorQueue,\n encryptedCustomFieldKeysResolver,\n })\n\n // Register in container\n container.register({\n searchService: asValue(searchService),\n searchStrategies: asValue(strategies),\n searchIndexer: asValue(searchIndexer),\n })\n}\n\n/**\n * Determine default strategy order based on available strategies.\n * Prefers fulltext > vector > tokens.\n */\nfunction determineDefaultStrategies(strategies: SearchStrategy[]): SearchStrategyId[] {\n const available = new Set(strategies.map((s) => s.id))\n const defaults: SearchStrategyId[] = []\n\n if (available.has('fulltext')) defaults.push('fulltext')\n if (available.has('vector')) defaults.push('vector')\n if (available.has('tokens')) defaults.push('tokens')\n\n return defaults.length > 0 ? defaults : ['tokens']\n}\n\n/**\n * Helper to add a custom strategy to an existing SearchService.\n *\n * @param container - DI container\n * @param strategy - Strategy to add\n */\nexport function addSearchStrategy(container: SearchContainer, strategy: SearchStrategy): void {\n const service = container.resolve<SearchService>('searchService')\n service.registerStrategy(strategy)\n\n const strategies = container.resolve<SearchStrategy[]>('searchStrategies')\n strategies.push(strategy)\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,oCAAoC;AAC7C,SAAS,eAAe;AAGxB,SAAS,qBAAqB;AAC9B,SAAS,2BAA2B;AACpC,SAAS,4BAAmD;AAC5D,SAAS,8BAA8B;AACvC,SAAS,4BAA4B;AACrC,SAAS,qBAAqB;AAkB9B,SAAS,+BAA+B;AACxC,SAAS,mCAAmC;AAC5C;AAAA,EACE;AAAA,OAGK;AACP,SAAS,yBAAyB;AAElC,SAAS,+BAAAA,oCAAmC;AAC5C;AAAA,EACE,0CAAAC;AAAA,OAEK;AAMP,MAAM,sBAAsB;AAM5B,SAAS,yBAA8C;AACrD,SAAO;AACT;AAOA,SAAS,+BAAwC;AAC/C,QAAM,OAAO,QAAQ,IAAI,mCAAmC,IAAI,YAAY;AAC5E,SAAO,QAAQ,OAAO,QAAQ,UAAU,QAAQ,SAAS,QAAQ;AACnE;AAEA,SAAS,oDAAsF;AAC7F,SAAO,YAAY;AACjB,UAAM,IAAI,MAAM,4GAA4G;AAAA,EAC9H;AACF;AAEA,SAAS,yCAAgG;AACvG,SAAO,YAAY;AACjB,UAAM,IAAI,MAAM,iGAAiG;AAAA,EACnH;AACF;AAsCO,SAAS,qBACd,WACA,SACM;AACN,QAAM,iBAAiB,SAAS,iBAAiB,CAAC,GAAG,IAAI,CAAC,kBAAkB;AAAA,IAC1E,GAAG;AAAA,IACH,UAAU,aAAa,SAAS,IAAI,CAAC,iBAAiB;AACpD,YAAM,cAAc,aAAa;AACjC,aAAO,cACH,EAAE,GAAG,cAAc,aAAa,CAAC,QAA2C,YAAY,KAAK,cAAc,EAAE,GAAG,KAAK,UAAU,CAAC,EAAE,IAClI;AAAA,IACN,CAAC;AAAA,EACH,EAAE;AACF,QAAM,aAA+B,CAAC;AAGtC,MAAI,CAAC,SAAS,YAAY;AACxB,QAAI;AACF,YAAM,KAAK,UAAU,QAAa,IAAI;AACtC,YAAM,KAAK,GAAG,UAAU;AACxB,iBAAW,KAAK,IAAI,oBAAoB,EAAE,CAAC;AAAA,IAC7C,QAAQ;AAAA,IAER;AAAA,EACF;AAKA,MAAI,CAAC,SAAS,YAAY;AACxB,QAAI;AACF,YAAM,mBAAmB,UAAU,QAA0B,wBAAwB;AACrF,YAAM,UAAU,UAAU,QAAwB,eAAe;AACjE,YAAM,gBAAgB,UAAU,CAAC;AAEjC,UAAI,oBAAoB,eAAe;AACrC,mBAAW,KAAK,IAAI,qBAAqB,kBAAkB,aAAa,CAAC;AAAA,MAC3E;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AAGA,QAAM,kBAAkB,oBAAI,IAAkC;AAC9D,aAAW,gBAAgB,eAAe;AACxC,eAAW,gBAAgB,aAAa,UAAU;AAChD,UAAI,aAAa,YAAY,OAAO;AAClC,wBAAgB,IAAI,aAAa,UAAsB,YAAY;AAAA,MACrE;AAAA,IACF;AAAA,EACF;AAGA,MAAI,CAAC,SAAS,cAAc;AAC1B,UAAM,mBAAmB,6BAA6B;AACtD,UAAM,wBAAwB,QAAQ,IAAI,mCAAmC;AAC7E,UAAM,IAAI,uBAAuB;AAMjC,UAAM,aAAa,yBAAyB,CAAC;AAE7C,QAAI,iBAA8C,aAC7C,EAAE,mBAAmB,KAAK,OAC3B;AAEJ,QAAI,CAAC,gBAAgB;AACnB,UAAI;AACJ,UAAI,kBAAkB;AACpB,YAAI;AACF,gBAAM,KAAK,UAAU,QAAa,IAAI;AACtC,gBAAM,KAAK,GAAG,UAAU;AACxB,kCAAwB,4BAA4B,EAAE;AAAA,QACxD,QAAQ;AACN,kCAAwB,uCAAuC;AAAA,QACjE;AAAA,MACF;AAEA,uBAAiB,qBAAqB;AAAA,QACpC,qBAAqB,CAAC,aAAsD;AAC1E,gBAAM,SAAS,gBAAgB,IAAI,QAAQ;AAC3C,iBAAO,QAAQ;AAAA,QACjB;AAAA,QACA;AAAA,MACF,CAAC;AAED,UAAI,cAAc,gBAAgB;AAChC,UAAE,mBAAmB,IAAI;AAAA,MAC3B;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,iBAAW,KAAK,IAAI,uBAAuB,cAAc,CAAC;AAAA,IAC5D;AAAA,EACF;AAGA,QAAM,oBAAoB,SAAS,qBAAqB,2BAA2B,UAAU;AAG7F,MAAI;AACJ,MAAI;AACF,kBAAc,UAAU,QAAqB,aAAa;AAAA,EAC5D,QAAQ;AAAA,EAER;AAGA,MAAI,oBAAwD;AAC5D,MAAI;AACF,wBAAoB,UAAU,QAAqC,yBAAyB;AAAA,EAC9F,QAAQ;AAAA,EAER;AAGA,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,UAAU,QAAa,IAAI;AACtC,UAAM,KAAK,GAAG,UAAU;AACxB,wBAAoB,wBAAwB,IAAI,iBAAiB,aAAa,iBAAiB;AAAA,EACjG,QAAQ;AAAA,EAER;AAGA,QAAM,gBAAgB,IAAI,cAAc;AAAA,IACtC;AAAA,IACA,yBAAyB,CAAC,aAAa,6BAA6B,UAAU,aAAa;AAAA,IAC3F;AAAA,IACA,kBAAkB;AAAA,IAClB,aAAa,SAAS,eAAe;AAAA,MACnC,mBAAmB;AAAA,MACnB,iBAAiB;AAAA,QACf,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,IACA;AAAA,EACF,CAAC;AAGD,MAAI;AACJ,MAAI;AACF,oBAAgB,UAAU,QAAwC,oBAAoB;AAAA,EACxF,QAAQ;AAAA,EAER;AAGA,MAAI;AACJ,MAAI;AACF,kBAAc,UAAU,QAAsC,kBAAkB;AAAA,EAClF,QAAQ;AAAA,EAER;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,UAAU,QAA6B,IAAI;AACtD,uCAAmC;AAAA,MACjC,GAAG,UAA0C;AAAA,IAC/C;AAAA,EACF,SAAS,OAAO;AACd,sBAAkB,gBAAgB,OAAO,6CAA6C;AACtF,uCAAmC,kDAAkD;AAAA,EACvF;AAEA,QAAM,gBAAgB,IAAI,cAAc,eAAe,eAAe;AAAA,IACpE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AAGD,YAAU,SAAS;AAAA,IACjB,eAAe,QAAQ,aAAa;AAAA,IACpC,kBAAkB,QAAQ,UAAU;AAAA,IACpC,eAAe,QAAQ,aAAa;AAAA,EACtC,CAAC;AACH;AAMA,SAAS,2BAA2B,YAAkD;AACpF,QAAM,YAAY,IAAI,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;AACrD,QAAM,WAA+B,CAAC;AAEtC,MAAI,UAAU,IAAI,UAAU,EAAG,UAAS,KAAK,UAAU;AACvD,MAAI,UAAU,IAAI,QAAQ,EAAG,UAAS,KAAK,QAAQ;AACnD,MAAI,UAAU,IAAI,QAAQ,EAAG,UAAS,KAAK,QAAQ;AAEnD,SAAO,SAAS,SAAS,IAAI,WAAW,CAAC,QAAQ;AACnD;AAQO,SAAS,kBAAkB,WAA4B,UAAgC;AAC5F,QAAM,UAAU,UAAU,QAAuB,eAAe;AAChE,UAAQ,iBAAiB,QAAQ;AAEjC,QAAM,aAAa,UAAU,QAA0B,kBAAkB;AACzE,aAAW,KAAK,QAAQ;AAC1B;",
|
|
6
|
+
"names": ["createEncryptionMapResolver", "createEncryptedCustomFieldKeysResolver"]
|
|
7
7
|
}
|
|
@@ -1,6 +1,60 @@
|
|
|
1
1
|
import { isEntityIndexStrategyAllowed } from "../lib/entity-strategy-policy.js";
|
|
2
|
-
import { searchDebug, searchDebugWarn, searchError,
|
|
2
|
+
import { reportSearchError, searchDebug, searchDebugWarn, searchError, searchWarn } from "../lib/debug.js";
|
|
3
|
+
import { normalizeDefinitionKey } from "@open-mercato/shared/lib/crud/custom-field-definition-index";
|
|
3
4
|
const MAX_PAGES = 1e4;
|
|
5
|
+
function toPlaintextExcludedCustomFields(exclusion) {
|
|
6
|
+
if (!exclusion) return void 0;
|
|
7
|
+
return exclusion === "all" ? "all" : Array.from(exclusion);
|
|
8
|
+
}
|
|
9
|
+
function mergeCustomFieldExclusions(exclusions) {
|
|
10
|
+
if (exclusions.includes("all")) return "all";
|
|
11
|
+
const sets = exclusions.filter((exclusion) => exclusion instanceof Set);
|
|
12
|
+
if (!sets.length) return null;
|
|
13
|
+
if (sets.length === 1) return sets[0];
|
|
14
|
+
return new Set(sets.flatMap((set) => Array.from(set)));
|
|
15
|
+
}
|
|
16
|
+
function isExcludedCustomField(key, exclusion) {
|
|
17
|
+
if (!exclusion) return false;
|
|
18
|
+
if (exclusion === "all") return true;
|
|
19
|
+
return exclusion.has(normalizeDefinitionKey(key));
|
|
20
|
+
}
|
|
21
|
+
function withoutExcludedRecordCustomFields(record, exclusion) {
|
|
22
|
+
if (!exclusion) return record;
|
|
23
|
+
const filtered = {};
|
|
24
|
+
let removed = false;
|
|
25
|
+
for (const [key, value] of Object.entries(record)) {
|
|
26
|
+
const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
|
|
27
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) {
|
|
28
|
+
removed = true;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
filtered[key] = value;
|
|
32
|
+
}
|
|
33
|
+
return removed ? filtered : record;
|
|
34
|
+
}
|
|
35
|
+
function applyBuildRecordChanges(buildRecord, record, exclusion) {
|
|
36
|
+
if (buildRecord === record) return;
|
|
37
|
+
for (const [key, value] of Object.entries(buildRecord)) {
|
|
38
|
+
if (record[key] !== value) record[key] = value;
|
|
39
|
+
}
|
|
40
|
+
for (const key of Object.keys(record)) {
|
|
41
|
+
if (key in buildRecord) continue;
|
|
42
|
+
const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
|
|
43
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue;
|
|
44
|
+
delete record[key];
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function withoutExcludedCustomFields(customFields, exclusion) {
|
|
48
|
+
if (!exclusion) return customFields;
|
|
49
|
+
const filtered = {};
|
|
50
|
+
for (const [key, value] of Object.entries(customFields)) {
|
|
51
|
+
const isPrefixed = key.startsWith("cf:") || key.startsWith("cf_");
|
|
52
|
+
if (isExcludedCustomField(key, exclusion)) continue;
|
|
53
|
+
if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue;
|
|
54
|
+
filtered[key] = value;
|
|
55
|
+
}
|
|
56
|
+
return filtered;
|
|
57
|
+
}
|
|
4
58
|
class SearchIndexer {
|
|
5
59
|
constructor(searchService, moduleConfigs, options) {
|
|
6
60
|
this.searchService = searchService;
|
|
@@ -10,6 +64,14 @@ class SearchIndexer {
|
|
|
10
64
|
this.fulltextQueue = options?.fulltextQueue;
|
|
11
65
|
this.vectorQueue = options?.vectorQueue;
|
|
12
66
|
this.encryptedCustomFieldKeysResolver = options?.encryptedCustomFieldKeysResolver;
|
|
67
|
+
if (!this.encryptedCustomFieldKeysResolver) {
|
|
68
|
+
searchWarn("SearchIndexer", "No encrypted custom-field resolver; excluding every custom field from build contexts");
|
|
69
|
+
reportSearchError(
|
|
70
|
+
"SearchIndexer",
|
|
71
|
+
new Error("[internal] SearchIndexer has no encryptedCustomFieldKeysResolver; excluding every custom field from build contexts"),
|
|
72
|
+
"search.encrypted_custom_fields_resolver_missing"
|
|
73
|
+
);
|
|
74
|
+
}
|
|
13
75
|
for (const moduleConfig of moduleConfigs) {
|
|
14
76
|
for (const entityConfig of moduleConfig.entities) {
|
|
15
77
|
if (entityConfig.enabled !== false) {
|
|
@@ -18,18 +80,30 @@ class SearchIndexer {
|
|
|
18
80
|
}
|
|
19
81
|
}
|
|
20
82
|
}
|
|
21
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Returns a lookup of the custom fields to keep out of build contexts, memoized per entity and
|
|
85
|
+
* tenant for one indexing call or reindex run. A missing resolver or a failed resolution excludes
|
|
86
|
+
* every custom field. Failed resolutions stay cached for the current operation and are retried by
|
|
87
|
+
* the fresh lookup created for the next top-level indexing call.
|
|
88
|
+
*/
|
|
89
|
+
createCustomFieldExclusionLookup() {
|
|
22
90
|
const resolver = this.encryptedCustomFieldKeysResolver;
|
|
23
91
|
const cache = /* @__PURE__ */ new Map();
|
|
24
|
-
return (entityId) => {
|
|
25
|
-
if (!resolver) return Promise.resolve(
|
|
26
|
-
|
|
92
|
+
return (entityId, tenantId) => {
|
|
93
|
+
if (!resolver) return Promise.resolve("all");
|
|
94
|
+
const cacheKey = `${entityId}\0${tenantId ?? ""}`;
|
|
95
|
+
let pending = cache.get(cacheKey);
|
|
27
96
|
if (!pending) {
|
|
28
|
-
pending = Promise.resolve().then(() => resolver(entityId
|
|
29
|
-
|
|
97
|
+
pending = Promise.resolve().then(() => resolver(entityId, tenantId)).catch((error) => {
|
|
98
|
+
const resolutionError = error instanceof Error ? error : new Error("[internal] Encrypted custom-field resolver rejected with a non-Error value");
|
|
99
|
+
searchWarn("SearchIndexer", "Encrypted custom fields could not be resolved; excluding every custom field", {
|
|
100
|
+
entityId,
|
|
101
|
+
error: resolutionError.message
|
|
102
|
+
});
|
|
103
|
+
reportSearchError("SearchIndexer", resolutionError, "search.encrypted_custom_fields_unresolved", { entityId });
|
|
30
104
|
return "all";
|
|
31
105
|
});
|
|
32
|
-
cache.set(
|
|
106
|
+
cache.set(cacheKey, pending);
|
|
33
107
|
}
|
|
34
108
|
return pending;
|
|
35
109
|
};
|
|
@@ -37,12 +111,27 @@ class SearchIndexer {
|
|
|
37
111
|
/**
|
|
38
112
|
* Returns a wrapped QueryEngine that forces skipAutoReindex: true on every query.
|
|
39
113
|
* Used to prevent feedback loops when search indexing callbacks (buildSource, formatResult)
|
|
40
|
-
* call queryEngine.query() to hydrate related entities.
|
|
114
|
+
* call queryEngine.query() to hydrate related entities. Encrypted custom fields are removed
|
|
115
|
+
* from the hydrated records, as from the indexed record itself, including those joined in
|
|
116
|
+
* from `customFieldSources`. A query without a tenant is checked against every tenant's definitions.
|
|
41
117
|
*/
|
|
42
|
-
|
|
118
|
+
createBuildQueryEngine(exclusions) {
|
|
43
119
|
if (!this.queryEngine) return void 0;
|
|
120
|
+
const queryEngine = this.queryEngine;
|
|
44
121
|
const wrapped = {
|
|
45
|
-
query: (entity, opts) =>
|
|
122
|
+
query: async (entity, opts) => {
|
|
123
|
+
const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true });
|
|
124
|
+
const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)];
|
|
125
|
+
const tenantId = opts?.tenantId ?? null;
|
|
126
|
+
const exclusion = mergeCustomFieldExclusions(
|
|
127
|
+
await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId, tenantId)))
|
|
128
|
+
);
|
|
129
|
+
if (!exclusion || !Array.isArray(result?.items)) return result;
|
|
130
|
+
return {
|
|
131
|
+
...result,
|
|
132
|
+
items: result.items.map((item) => withoutExcludedRecordCustomFields(item, exclusion))
|
|
133
|
+
};
|
|
134
|
+
}
|
|
46
135
|
};
|
|
47
136
|
return wrapped;
|
|
48
137
|
}
|
|
@@ -69,22 +158,26 @@ class SearchIndexer {
|
|
|
69
158
|
* Index a record in the search service.
|
|
70
159
|
*/
|
|
71
160
|
async indexRecord(params) {
|
|
72
|
-
|
|
161
|
+
await this.indexRecordWithExclusions(params, this.createCustomFieldExclusionLookup());
|
|
162
|
+
}
|
|
163
|
+
async indexRecordWithExclusions(params, exclusions) {
|
|
164
|
+
const indexableRecord = await this.buildIndexableRecord(params, exclusions);
|
|
73
165
|
if (indexableRecord) {
|
|
74
166
|
await this.searchService.index(indexableRecord);
|
|
75
167
|
}
|
|
76
168
|
}
|
|
77
|
-
async buildIndexableRecord(params, exclusions = this.
|
|
169
|
+
async buildIndexableRecord(params, exclusions = this.createCustomFieldExclusionLookup()) {
|
|
78
170
|
const config = this.entityConfigMap.get(params.entityId);
|
|
79
171
|
if (!config || config.enabled === false) {
|
|
80
172
|
return null;
|
|
81
173
|
}
|
|
174
|
+
const exclusion = await exclusions(params.entityId, params.tenantId);
|
|
82
175
|
const buildContext = {
|
|
83
|
-
record: params.record,
|
|
84
|
-
customFields: params.customFields ?? {},
|
|
176
|
+
record: withoutExcludedRecordCustomFields(params.record, exclusion),
|
|
177
|
+
customFields: withoutExcludedCustomFields(params.customFields ?? {}, exclusion),
|
|
85
178
|
organizationId: params.organizationId,
|
|
86
179
|
tenantId: params.tenantId,
|
|
87
|
-
queryEngine: this.
|
|
180
|
+
queryEngine: this.createBuildQueryEngine(exclusions)
|
|
88
181
|
};
|
|
89
182
|
let text = config.buildSource ? [] : void 0;
|
|
90
183
|
let presenter;
|
|
@@ -144,13 +237,14 @@ class SearchIndexer {
|
|
|
144
237
|
});
|
|
145
238
|
}
|
|
146
239
|
}
|
|
240
|
+
applyBuildRecordChanges(buildContext.record, params.record, exclusion);
|
|
147
241
|
return {
|
|
148
242
|
entityId: params.entityId,
|
|
149
243
|
recordId: params.recordId,
|
|
150
244
|
tenantId: params.tenantId,
|
|
151
245
|
organizationId: params.organizationId,
|
|
152
246
|
fields: params.record,
|
|
153
|
-
plaintextExcludedCustomFields:
|
|
247
|
+
plaintextExcludedCustomFields: toPlaintextExcludedCustomFields(exclusion),
|
|
154
248
|
presenter,
|
|
155
249
|
url,
|
|
156
250
|
links,
|
|
@@ -163,6 +257,9 @@ class SearchIndexer {
|
|
|
163
257
|
* Used by workers that only have record identifiers.
|
|
164
258
|
*/
|
|
165
259
|
async indexRecordById(params) {
|
|
260
|
+
return this.indexRecordByIdWithExclusions(params, this.createCustomFieldExclusionLookup());
|
|
261
|
+
}
|
|
262
|
+
async indexRecordByIdWithExclusions(params, exclusions) {
|
|
166
263
|
if (!this.queryEngine) {
|
|
167
264
|
return { action: "skipped", reason: "queryEngine not available" };
|
|
168
265
|
}
|
|
@@ -189,14 +286,14 @@ class SearchIndexer {
|
|
|
189
286
|
customFields[key.slice(3)] = value;
|
|
190
287
|
}
|
|
191
288
|
}
|
|
192
|
-
await this.
|
|
289
|
+
await this.indexRecordWithExclusions({
|
|
193
290
|
entityId: params.entityId,
|
|
194
291
|
recordId: params.recordId,
|
|
195
292
|
tenantId: params.tenantId,
|
|
196
293
|
organizationId: params.organizationId,
|
|
197
294
|
record,
|
|
198
295
|
customFields
|
|
199
|
-
});
|
|
296
|
+
}, exclusions);
|
|
200
297
|
return { action: "indexed" };
|
|
201
298
|
} catch (error) {
|
|
202
299
|
searchError("SearchIndexer", "Failed to load record for indexing", {
|
|
@@ -226,7 +323,7 @@ class SearchIndexer {
|
|
|
226
323
|
list.push(item.recordId);
|
|
227
324
|
recordIdsByEntity.set(item.entityId, list);
|
|
228
325
|
}
|
|
229
|
-
const exclusions = this.
|
|
326
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
230
327
|
const allRecords = [];
|
|
231
328
|
let skipped = 0;
|
|
232
329
|
for (const [entityId, recordIds] of recordIdsByEntity) {
|
|
@@ -325,6 +422,7 @@ class SearchIndexer {
|
|
|
325
422
|
});
|
|
326
423
|
}
|
|
327
424
|
}
|
|
425
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
328
426
|
let page = 1;
|
|
329
427
|
const pageSize = 200;
|
|
330
428
|
let hasMore = true;
|
|
@@ -347,7 +445,8 @@ class SearchIndexer {
|
|
|
347
445
|
params.tenantId,
|
|
348
446
|
params.organizationId ?? null,
|
|
349
447
|
items,
|
|
350
|
-
config
|
|
448
|
+
config,
|
|
449
|
+
exclusions
|
|
351
450
|
);
|
|
352
451
|
if (records.length > 0) {
|
|
353
452
|
try {
|
|
@@ -406,7 +505,7 @@ class SearchIndexer {
|
|
|
406
505
|
*/
|
|
407
506
|
async bulkIndexRecords(params) {
|
|
408
507
|
const indexableRecords = [];
|
|
409
|
-
const exclusions = this.
|
|
508
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
410
509
|
for (const param of params) {
|
|
411
510
|
const indexableRecord = await this.buildIndexableRecord(param, exclusions);
|
|
412
511
|
if (indexableRecord) {
|
|
@@ -485,6 +584,7 @@ class SearchIndexer {
|
|
|
485
584
|
if (params.recreateIndex !== false) {
|
|
486
585
|
await fulltext.recreateIndex(params.tenantId);
|
|
487
586
|
}
|
|
587
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
488
588
|
const pageSize = 200;
|
|
489
589
|
let page = 1;
|
|
490
590
|
let totalProcessed = 0;
|
|
@@ -516,7 +616,8 @@ class SearchIndexer {
|
|
|
516
616
|
params.tenantId,
|
|
517
617
|
params.organizationId ?? null,
|
|
518
618
|
queryResult.items,
|
|
519
|
-
config
|
|
619
|
+
config,
|
|
620
|
+
exclusions
|
|
520
621
|
);
|
|
521
622
|
result.recordsDropped = (result.recordsDropped ?? 0) + dropped;
|
|
522
623
|
if (indexableRecords.length > 0) {
|
|
@@ -682,6 +783,7 @@ class SearchIndexer {
|
|
|
682
783
|
let page = 1;
|
|
683
784
|
let totalProcessed = 0;
|
|
684
785
|
let jobsEnqueued = 0;
|
|
786
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
685
787
|
for (; ; ) {
|
|
686
788
|
params.onProgress?.({
|
|
687
789
|
entityId: params.entityId,
|
|
@@ -732,12 +834,12 @@ class SearchIndexer {
|
|
|
732
834
|
} else {
|
|
733
835
|
for (const { entityId, recordId } of batchRecords) {
|
|
734
836
|
try {
|
|
735
|
-
await this.
|
|
837
|
+
await this.indexRecordByIdWithExclusions({
|
|
736
838
|
entityId,
|
|
737
839
|
recordId,
|
|
738
840
|
tenantId: params.tenantId,
|
|
739
841
|
organizationId: params.organizationId
|
|
740
|
-
});
|
|
842
|
+
}, exclusions);
|
|
741
843
|
totalProcessed++;
|
|
742
844
|
} catch (error) {
|
|
743
845
|
searchDebugWarn("SearchIndexer", "Failed to index record to vector", {
|
|
@@ -818,20 +920,17 @@ class SearchIndexer {
|
|
|
818
920
|
* Build IndexableRecords from raw query results.
|
|
819
921
|
* Returns records and count of dropped items (missing id or other validation failures).
|
|
820
922
|
*/
|
|
821
|
-
async buildIndexableRecords(entityId, tenantId, organizationId, items, config, exclusions = this.
|
|
923
|
+
async buildIndexableRecords(entityId, tenantId, organizationId, items, config, exclusions = this.createCustomFieldExclusionLookup()) {
|
|
822
924
|
const records = [];
|
|
823
|
-
const plaintextExcludedCustomFields = await exclusions(entityId);
|
|
824
925
|
let dropped = 0;
|
|
926
|
+
const exclusion = await exclusions(entityId, tenantId);
|
|
927
|
+
const plaintextExcludedCustomFields = toPlaintextExcludedCustomFields(exclusion);
|
|
928
|
+
const buildQueryEngine = this.createBuildQueryEngine(exclusions);
|
|
825
929
|
if (items.length > 0) {
|
|
826
930
|
searchDebug("SearchIndexer", "Sample item structure", {
|
|
827
931
|
entityId,
|
|
828
932
|
sampleKeys: Object.keys(items[0]),
|
|
829
|
-
|
|
830
|
-
hasId: "id" in items[0],
|
|
831
|
-
firstName: items[0].first_name,
|
|
832
|
-
lastName: items[0].last_name,
|
|
833
|
-
preferredName: items[0].preferred_name,
|
|
834
|
-
sampleItem: JSON.stringify(items[0]).slice(0, 500)
|
|
933
|
+
hasId: "id" in items[0]
|
|
835
934
|
});
|
|
836
935
|
}
|
|
837
936
|
for (const item of items) {
|
|
@@ -849,11 +948,11 @@ class SearchIndexer {
|
|
|
849
948
|
}
|
|
850
949
|
}
|
|
851
950
|
const buildContext = {
|
|
852
|
-
record: item,
|
|
853
|
-
customFields,
|
|
951
|
+
record: withoutExcludedRecordCustomFields(item, exclusion),
|
|
952
|
+
customFields: withoutExcludedCustomFields(customFields, exclusion),
|
|
854
953
|
organizationId,
|
|
855
954
|
tenantId,
|
|
856
|
-
queryEngine:
|
|
955
|
+
queryEngine: buildQueryEngine
|
|
857
956
|
};
|
|
858
957
|
let text = config.buildSource ? [] : void 0;
|
|
859
958
|
let presenter;
|
|
@@ -898,6 +997,7 @@ class SearchIndexer {
|
|
|
898
997
|
} catch {
|
|
899
998
|
}
|
|
900
999
|
}
|
|
1000
|
+
applyBuildRecordChanges(buildContext.record, item, exclusion);
|
|
901
1001
|
records.push({
|
|
902
1002
|
entityId,
|
|
903
1003
|
recordId,
|