@memberjunction/search-engine 5.33.0 → 5.34.0

package/dist/generic/SearchEngine.js

@@ -5,7 +5,14 @@
  * fuses results with Reciprocal Rank Fusion (RRF), applies enrichment
  * (entity icons, record names, tags), and filters by minimum score.
  *
- * Providers are loaded via @RegisterClass(BaseSearchProvider, DriverClass) and
+ * When `SearchParams.ScopeIDs` is provided, the engine resolves each scope against
+ * `SearchEngineBase` (which caches all `MJ: Search Scope*` metadata), builds a
+ * `ScopeConstraints` object per scope (including Nunjucks-rendered MetadataFilter,
+ * ExtraFilter, UserSearchString, and FolderPath values), runs each scope's providers
+ * in parallel, then fuses the per-scope results via cross-scope RRF. An optional
+ * re-ranker stage (`BaseReRanker`) runs after fusion when configured.
+ *
+ * Providers are loaded via `@RegisterClass(BaseSearchProvider, DriverClass)` and
  * instantiated using the MJ ClassFactory based on active SearchProvider records.
  *
  * Uses BaseSingleton from @memberjunction/global for a truly global instance.
@@ -14,29 +21,43 @@
  */
 import { EntityPermissionType, LogError, LogStatus, RunView } from '@memberjunction/core';
 import { SearchEngineBase } from '@memberjunction/core-entities';
-import { BaseSingleton, MJGlobal, NormalizeUUID } from '@memberjunction/global';
+import { BaseSingleton, MJGlobal, NormalizeUUID, UUIDsEqual } from '@memberjunction/global';
 import { BaseSearchProvider } from './ISearchProvider.js';
 import { SearchFusion } from './SearchFusion.js';
 import { SearchEnricher } from './SearchEnricher.js';
 import { FullTextSearchProvider } from './FullTextSearchProvider.js';
+import { BaseReRanker } from './BaseReRanker.js';
+import { NoopReRanker, LoadNoopReRanker } from './NoopReRanker.js';
+import { RerankerBudgetGuard } from '../rerankers/RerankerBudgetGuard.js';
+import { RenderScopeTemplate, RenderScopeJsonTemplate } from './ScopeTemplateRenderer.js';
+// Keep the default re-ranker registration alive under tree-shaking
+LoadNoopReRanker();
 /**
  * Singleton search engine that orchestrates multi-source search with RRF fusion.
  *
  * Providers are discovered from the MJ: Search Providers entity. Each active
  * provider's DriverClass is resolved via ClassFactory to create an instance,
- * which is then initialized with the provider's config from the DB.
+ * which is then initialized with the provider's config from the DB record.
  *
  * Usage:
  * ```typescript
  * // Initialize once at server startup
  * await SearchEngine.Instance.Config({}, contextUser);
  *
- * // Execute searches
+ * // Execute searches (unscoped — original behavior)
  * const result = await SearchEngine.Instance.Search({
  *     Query: 'quarterly revenue',
  *     MaxResults: 20,
  *     MinScore: 0.1
  * }, contextUser);
+ *
+ * // Scoped search against two scopes with multi-tenant context
+ * const scopedResult = await SearchEngine.Instance.Search({
+ *     Query: 'refund policy',
+ *     MaxResults: 20,
+ *     ScopeIDs: ['hr-scope-id', 'legal-scope-id'],
+ *     SearchContext: { PrimaryScopeRecordID: 'tenant-a' }
+ * }, contextUser);
  * ```
  */
 export class SearchEngine extends BaseSingleton {
@@ -48,6 +69,7 @@ export class SearchEngine extends BaseSingleton {
         this._fusion = new SearchFusion();
         this._enricher = new SearchEnricher();
         this._defaultMaxResults = 20;
+        this._defaultOverfetchFactor = 2;
         this._cache = new Map();
     }
     /** Static accessor for the global singleton instance */
@@ -92,8 +114,9 @@ export class SearchEngine extends BaseSingleton {
         if (this._configured && !forceRefresh)
             return;
         this._defaultMaxResults = config.DefaultMaxResults ?? 20;
+        this._defaultOverfetchFactor = Math.max(1, config.DefaultPermissionOverfetchFactor ?? 2);
         this._providerEntries = [];
-        // Ensure SearchEngineBase has loaded provider metadata
+        // Ensure SearchEngineBase has loaded provider + scope metadata
         await this.Base.Config(forceRefresh, contextUser);
         const providerRecords = this.Base.ActiveProviders;
         if (providerRecords.length === 0) {
@@ -111,30 +134,54 @@ export class SearchEngine extends BaseSingleton {
         this._providerEntries.sort((a, b) => a.Priority - b.Priority);
         this._configured = true;
         const names = this._providerEntries.map(e => e.Provider.SourceType);
-        LogStatus(`SearchEngine: Configured with ${this._providerEntries.length} provider(s): ${names.join(', ')}`);
+        LogStatus(`SearchEngine: Configured with ${this._providerEntries.length} provider(s): ${names.join(', ')} and ${this.Base.Scopes.length} scope(s)`);
     }
     /**
      * Execute a multi-source search with RRF fusion and optional enrichment.
      *
-     * Steps:
-     * 1. Run all available providers in parallel
-     * 2. Fuse results with RRF
-     * 3. Deduplicate by EntityName+RecordID
-     * 4. Exclude redundant entity-sourced Content Items
-     * 5. Apply minimum score threshold
-     * 6. Enrich with icons, names, and tags (skipped in preview mode)
+     * When `params.ScopeIDs` is provided, each scope runs independently and the results
+     * are combined via cross-scope RRF before deduplication, re-ranking, and enrichment.
      *
      * @param params - Search parameters
      * @param contextUser - The user performing the search
      * @returns Aggregated search result
      */
     async Search(params, contextUser) {
+        return this.searchInternal(params, contextUser);
+    }
+    /**
+     * Internal search implementation that optionally fires `onProviderResolved`
+     * as each provider's promise settles. Exposed via the public {@link Search}
+     * (no callback) and {@link streamSearch} (queue-backed callback that
+     * yields `provider` events to the caller).
+     */
+    async searchInternal(params, contextUser, onProviderResolved) {
         const startTime = Date.now();
+        // Per-invocation tracking for the post-search SearchExecutionLog row (P3.2).
+        let invocationBudgetGuard = null;
+        let invocationRerankerName = null;
         try {
-            const trimmed = (params.Query ?? '').trim();
-            if (!trimmed) {
+            // Defensive null-check: `params.Query.trim()` throws on null/undefined,
+            // and Sage's LLM has been observed to emit empty/missing tool args.
+            // Coerce to string before validating so we surface a clean error
+            // instead of a TypeError that would also skip the audit-log row.
+            if (params.Query == null || typeof params.Query !== 'string' || !params.Query.trim()) {
+                this.logSearchExecution({
+                    Status: 'Failure',
+                    FailureReason: 'Query cannot be empty',
+                    Query: typeof params.Query === 'string' ? params.Query : '',
+                    ScopeIDs: params.ScopeIDs,
+                    StartTime: startTime,
+                    ResultCount: 0,
+                    RerankerName: null,
+                    RerankerCostCents: null,
+                    SourceCounts: undefined,
+                    ContextUser: contextUser,
+                    AIAgentID: params.AIAgentID ?? null,
+                });
                 return this.buildErrorResult('Query cannot be empty', startTime);
             }
+            const trimmed = params.Query.trim();
             if (trimmed.length < SearchEngine.MIN_TERM_LENGTH) {
                 // Short queries hit unindexed table-scans fanned out across every
                 // searchable entity with negligible relevance. Return an empty
@@ -155,10 +202,19 @@ export class SearchEngine extends BaseSingleton {
             const topK = params.MaxResults ?? this._defaultMaxResults;
             const mode = params.Mode ?? 'full';
             const isPreview = mode === 'preview';
-            // Cache lookup. Skip preview searches — they're already cheap and
-            // caching would mask config changes during dev. Cache key includes
-            // the user's ID so two users with different RLS scopes never share
-            // an entry.
+            const overfetchFactor = Math.max(1, params.PermissionOverfetchFactor ?? this._defaultOverfetchFactor);
+            const providerTopK = Math.max(topK, Math.ceil(topK * overfetchFactor));
+            // ──────────────────────────────────────────────────────────
+            // Resolve scopes (when supplied)
+            // ──────────────────────────────────────────────────────────
+            const resolvedScopes = this.resolveScopes(params.ScopeIDs);
+            const isUnconstrained = resolvedScopes.length === 0 || resolvedScopes.some(s => s.Scope.IsGlobal);
+            // ──────────────────────────────────────────────────────────
+            // Cache lookup (next PR #2532). Skip preview searches — they're already
+            // cheap and caching would mask config changes during dev. Cache key
+            // includes the user's ID so two users with different RLS scopes never
+            // share an entry.
+            // ──────────────────────────────────────────────────────────
             const cacheKey = isPreview ? null : this.buildCacheKey(trimmed, params, contextUser);
             if (cacheKey) {
                 const hit = this._cache.get(cacheKey);
@@ -171,27 +227,88 @@ export class SearchEngine extends BaseSingleton {
                 if (hit)
                     this._cache.delete(cacheKey); // expired
             }
-            // Step 1: Run all providers in parallel (respecting preview flag)
-            const labeledLists = await this.executeProviders(trimmed, topK, params.Filters, contextUser, isPreview);
-            const sourceCounts = this.countSources(labeledLists);
-            // Step 2: Fuse with RRF
-            let results = this._fusion.Fuse(labeledLists, topK);
-            // Step 3: Deduplicate
-            results = this._fusion.Deduplicate(results);
-            // Step 4: Exclude redundant Content Items
+            // ──────────────────────────────────────────────────────────
+            // Execute providers — either unscoped (original path) or per-scope
+            // ──────────────────────────────────────────────────────────
+            let sourceCounts;
+            let fusedResults;
+            if (isUnconstrained) {
+                const labeledLists = await this.executeProviders(params.Query, providerTopK, params.Filters, contextUser, isPreview, undefined, onProviderResolved);
+                sourceCounts = this.countSources(labeledLists);
+                const defaultFusionWeights = params.FusionWeightsOverride;
+                fusedResults = this._fusion.Fuse(labeledLists, providerTopK, defaultFusionWeights);
+            }
+            else {
+                // Run each scope independently, then cross-scope RRF
+                const perScopeRunResults = await Promise.all(resolvedScopes.map(bundle => this.executeScopeBundle(params.Query, providerTopK, params.Filters, contextUser, isPreview, bundle, params.SearchContext, params.FusionWeightsOverride, onProviderResolved)));
+                const sc = { Vector: 0, FullText: 0, Entity: 0, Storage: 0 };
+                const perScopeFused = new Map();
+                for (const r of perScopeRunResults) {
+                    sc.Vector += r.sourceCounts.Vector;
+                    sc.FullText += r.sourceCounts.FullText;
+                    sc.Entity += r.sourceCounts.Entity;
+                    sc.Storage += r.sourceCounts.Storage;
+                    perScopeFused.set(r.scopeID, r.fused);
+                }
+                sourceCounts = sc;
+                if (perScopeFused.size > 1) {
+                    fusedResults = this._fusion.CrossScopeFusion(perScopeFused, providerTopK);
+                }
+                else {
+                    const only = perScopeFused.values().next().value;
+                    fusedResults = only ?? [];
+                }
+            }
+            // ──────────────────────────────────────────────────────────
+            // Optional re-ranker stage (one per leading scope — pick the first active scope's config)
+            // ──────────────────────────────────────────────────────────
+            const reRankerConfig = this.pickReRankerConfig(resolvedScopes);
+            if (reRankerConfig?.driverClass) {
+                // Budget guard (P2D.6): cap real-provider rerank spend at the scope's
+                // RerankerBudgetCents. Pulled from the same scope that supplied the
+                // reranker config — keeping the policy local to the scope that opted in.
+                const budgetCents = this.pickRerankerBudgetCents(resolvedScopes);
+                invocationBudgetGuard = new RerankerBudgetGuard(budgetCents);
+                invocationRerankerName = reRankerConfig.driverClass;
+                fusedResults = await this.runReRanker(params.Query, fusedResults, reRankerConfig, contextUser, invocationBudgetGuard);
+            }
+            // ──────────────────────────────────────────────────────────
+            // Dedup → content-item exclusion → permission safety net → score threshold → enrich
+            // ──────────────────────────────────────────────────────────
+            let results = this._fusion.Deduplicate(fusedResults);
             results = await this._enricher.ExcludeEntitySourcedContentItems(results, contextUser);
-            // Step 4.5: Filter by entity-level and row-level permissions
+            const beforePermCount = results.length;
             results = await this.filterByPermissions(results, contextUser);
-            // Step 5: Apply minimum score threshold
+            const lateFilteredCount = beforePermCount - results.length;
+            if (lateFilteredCount > 0) {
+                // Observability: Section 3.6 — if a provider's push-down is complete, this
+                // number should be zero (the safety net should never trim anything).
+                LogStatus(`SearchEngine: Residual permission filter removed ${lateFilteredCount} result(s) — consider tightening provider push-down.`);
+            }
             const scoreThreshold = params.MinScore ?? 0;
             if (scoreThreshold > 0) {
                 results = results.filter(r => r.Score >= scoreThreshold);
             }
-            // Step 6: Enrich (skip in preview mode)
+            // Trim to caller's requested topK (we overfetched earlier)
+            if (results.length > topK)
+                results = results.slice(0, topK);
             if (!isPreview) {
                 await this._enricher.Enrich(results, contextUser);
             }
-            LogStatus(`SearchEngine: Search complete in ${Date.now() - startTime}ms - ${results.length} results`);
+            LogStatus(`SearchEngine: Search complete in ${Date.now() - startTime}ms - ${results.length} result(s)${resolvedScopes.length ? ` across ${resolvedScopes.length} scope(s)` : ''}`);
+            this.logSearchExecution({
+                Status: 'Success',
+                FailureReason: null,
+                Query: params.Query,
+                ScopeIDs: params.ScopeIDs,
+                StartTime: startTime,
+                ResultCount: results.length,
+                RerankerName: invocationRerankerName,
+                RerankerCostCents: invocationBudgetGuard ? invocationBudgetGuard.Spent : null,
+                SourceCounts: sourceCounts,
+                ContextUser: contextUser,
+                AIAgentID: params.AIAgentID ?? null,
+            });
             const finalResult = {
                 Success: true,
                 Results: results,
@@ -208,9 +325,163 @@ export class SearchEngine extends BaseSingleton {
         catch (error) {
             const msg = error instanceof Error ? error.message : String(error);
             LogError(`SearchEngine: Search failed: ${msg}`);
+            this.logSearchExecution({
+                Status: 'Failure',
+                FailureReason: msg,
+                Query: params.Query,
+                ScopeIDs: params.ScopeIDs,
+                StartTime: startTime,
+                ResultCount: 0,
+                RerankerName: invocationRerankerName,
+                RerankerCostCents: invocationBudgetGuard ? invocationBudgetGuard.Spent : null,
+                SourceCounts: undefined,
+                ContextUser: contextUser,
+                AIAgentID: params.AIAgentID ?? null,
+            });
             return this.buildErrorResult(msg, startTime);
         }
     }
+    /**
+     * Streaming variant of {@link Search}. Yields events as each pipeline
+     * stage produces output so the caller can emit partials to the UI / agent
+     * before fusion + reranking complete.
+     *
+     * **Phase 2C v1 semantics:** runs the same internal pipeline as
+     * {@link Search} and emits synthetic events at each transition. This
+     * preserves all existing fusion / permission / dedup / enrich behavior
+     * — important because those steps have subtle correctness rules that
+     * we don't want to re-implement in a parallel code path. Per-provider
+     * partials are reconstructed from the final SourceCounts; a future
+     * refactor (Phase 2C v2) can split provider emission to true real-time
+     * concurrent emission once we measure that the synthetic phase is the
+     * actual bottleneck.
+     *
+     * Cancellation: the consumer can stop iterating at any point — the
+     * underlying Search() will run to completion but its result is
+     * discarded. AbortSignal-based mid-pipeline cancellation is a Phase 2C
+     * v2 concern.
+     *
+     * Event ordering:
+     *   1. Zero or more `provider` events (one per non-empty source)
+     *   2. Exactly one `fused` event
+     *   3. Optional one `reranked` event (when a reranker is configured)
+     *   4. Exactly one `final` event
+     *   5. On error: a single `error` event in place of `final`.
+     *
+     * @example
+     * for await (const ev of SearchEngine.Instance.streamSearch(params, user)) {
+     *   switch (ev.phase) {
+     *     case 'provider':  scratchpad.append(`${ev.providerName}: ${ev.results.length} hits`); break;
+     *     case 'final':     scratchpad.commit(ev.results); break;
+     *     case 'error':     scratchpad.fail(ev.error); break;
+     *   }
+     * }
+     */
+    async *streamSearch(params, contextUser) {
+        // Phase 2C v2: true concurrent emission. The internal search is run
+        // with an `onProviderResolved` callback that pushes a `provider`
+        // event into a queue the moment each provider's promise settles.
+        // The generator drains the queue while the search keeps running, so
+        // `provider` events arrive as fast as their providers resolve. After
+        // the search completes (or errors) we emit `fused` + `final` (or
+        // `error`) and close the iterator.
+        //
+        // Cancellation: if the consumer breaks out of `for await`, the
+        // generator's `return()` runs and the underlying search is allowed
+        // to finish in the background (its result is discarded). Mid-pipeline
+        // AbortSignal propagation is a future enhancement.
+        const queue = [];
+        let resolveNext = null;
+        let done = false;
+        const push = (ev) => {
+            queue.push(ev);
+            if (resolveNext) {
+                const r = resolveNext;
+                resolveNext = null;
+                r();
+            }
+        };
+        const finish = () => {
+            done = true;
+            if (resolveNext) {
+                const r = resolveNext;
+                resolveNext = null;
+                r();
+            }
+        };
+        // Source-type → friendly provider label mapping for stable UI display.
+        // Keys are lowercased to match what providers report.
+        const sourceTypeToLabel = {
+            vector: 'Vector',
+            fulltext: 'FullText',
+            entity: 'Entity',
+            storage: 'Storage',
+        };
+        const onProviderResolved = (ev) => {
+            const label = sourceTypeToLabel[ev.sourceType.toLowerCase()] ?? ev.sourceType;
+            push({
+                phase: 'provider',
+                providerName: label,
+                results: ev.results,
+                durationMs: ev.durationMs,
+            });
+        };
+        // Kick off the search; do NOT await it here — we want the generator
+        // loop below to interleave with the provider callbacks.
+        const searchPromise = (async () => {
+            try {
+                const result = await this.searchInternal(params, contextUser, onProviderResolved);
+                if (!result.Success) {
+                    push({ phase: 'error', error: result.ErrorMessage ?? 'Search failed' });
+                    return;
+                }
+                push({ phase: 'fused', results: result.Results });
+                // Reranker emission is intentionally elided here — the engine's
+                // post-fusion rerank fires inside `searchInternal` before this
+                // point, and observers seeking that signal should look at the
+                // final SearchExecutionLog row. Keeping this generator narrow
+                // avoids leaking rerank internals into a streaming surface that
+                // can't faithfully separate them from fusion.
+                push({
+                    phase: 'final',
+                    results: result.Results,
+                    sourceCounts: result.SourceCounts,
+                    elapsedMs: result.ElapsedMs,
+                });
+            }
+            catch (err) {
+                push({ phase: 'error', error: err instanceof Error ? err.message : String(err) });
+            }
+            finally {
+                finish();
+            }
+        })();
+        // Drain the queue. The loop blocks on `resolveNext` between bursts
+        // so that we don't busy-spin while waiting for providers.
+        try {
+            while (true) {
+                if (queue.length > 0) {
+                    yield queue.shift();
+                    continue;
+                }
+                if (done) {
+                    break;
+                }
+                await new Promise((resolve) => {
+                    resolveNext = resolve;
+                });
+            }
+        }
+        finally {
+            // If the consumer aborts mid-iteration, surface any background
+            // error so it isn't silently swallowed. We don't await the
+            // promise on the happy path because `done` already implies it
+            // settled.
+            if (!done) {
+                searchPromise.catch(() => { });
+            }
+        }
+    }
     /**
      * Build a stable cache key for a search. Includes the user identity so RLS
      * scopes never bleed across users, plus the trimmed query, MaxResults,
@@ -258,6 +529,239 @@ export class SearchEngine extends BaseSingleton {
         }, contextUser);
     }
     // ────────────────────────────────────────────────────────────────
+    // Scope resolution
+    // ────────────────────────────────────────────────────────────────
+    /**
+     * Load `ScopeBundle`s for each requested scope ID, filtering out inactive / expired.
+     * Returns an empty array when no scope IDs are supplied (caller treats as Global).
+     */
+    resolveScopes(scopeIDs) {
+        if (!scopeIDs || scopeIDs.length === 0)
+            return [];
+        const bundles = [];
+        for (const id of scopeIDs) {
+            const scope = this.Base.GetActiveScopeByID(id);
+            if (!scope) {
+                LogStatus(`SearchEngine: Requested scope "${id}" is not active or does not exist — skipping.`);
+                continue;
+            }
+            const bundle = this.Base.GetScopeBundle(id);
+            if (bundle)
+                bundles.push(bundle);
+        }
+        return bundles;
+    }
+    /**
+     * Execute all scoped providers for a single scope bundle and return per-scope fused results.
+     */
+    async executeScopeBundle(query, topK, filters, contextUser, isPreview, bundle, searchContext, agentFusionWeights, onProviderResolved) {
+        const scope = bundle.Scope;
+        const scopeConfig = this.parseJson(scope.ScopeConfig);
+        const constraints = this.buildScopeConstraints(bundle, searchContext);
+        const perProviderQueryTransforms = constraints.QueryTransforms ?? {};
+        // Determine which providers this scope participates in (SearchScopeProvider rows)
+        const scopeProviderIDs = new Set(bundle.Providers.map(p => NormalizeUUID(p.SearchProviderID)));
+        const allowAllProviders = scopeProviderIDs.size === 0; // empty = scope is IsGlobal or all-inclusive
+        const applicableProviders = this._providerEntries.filter(entry => {
+            if (!entry.Provider.IsAvailable())
+                return false;
+            if (isPreview && !entry.SupportsPreview)
+                return false;
+            if (allowAllProviders)
+                return true;
+            return scopeProviderIDs.has(NormalizeUUID(entry.ID));
+        });
+        if (applicableProviders.length === 0) {
+            LogStatus(`SearchEngine: Scope "${scope.Name}" has no applicable providers — skipping.`);
+            return {
+                scopeID: scope.ID,
+                fused: [],
+                sourceCounts: { Vector: 0, FullText: 0, Entity: 0, Storage: 0 }
+            };
+        }
+        // Resolve per-provider `SearchScopeProvider.MaxResultsOverride` if present
+        const promises = applicableProviders.map(async (entry) => {
+            const providerStart = Date.now();
+            try {
+                const spRow = bundle.Providers.find(r => UUIDsEqual(r.SearchProviderID, entry.ID));
+                const effectiveTopK = spRow?.MaxResultsOverride ?? entry.MaxResultsOverride ?? topK;
+                // If this provider has a per-provider QueryTransform override, stash it
+                // under the provider's SourceType in QueryTransforms so the provider finds it.
+                const perProviderConstraints = {
+                    ...constraints,
+                    QueryTransforms: { ...perProviderQueryTransforms }
+                };
+                // (Note: actual Nunjucks-rendered `QueryTransformTemplateID` resolution for
+                // stored templates lives in AgentPreExecutionRAG/ScopedSearchAction, not here.
+                // This engine only forwards already-rendered strings that the caller provides.)
+                const providerResults = await entry.Provider.Search(query, effectiveTopK, filters, contextUser, perProviderConstraints);
+                // Stamp provider metadata onto each result
+                for (const r of providerResults) {
+                    r.ProviderId = entry.ID;
+                    r.ProviderLabel = entry.DisplayName;
+                    r.ProviderIcon = entry.Icon;
+                }
+                if (onProviderResolved) {
+                    try {
+                        onProviderResolved({
+                            sourceType: entry.Provider.SourceType,
+                            results: providerResults,
+                            durationMs: Date.now() - providerStart,
+                            scopeID: scope.ID,
+                        });
+                    }
+                    catch (cbErr) {
+                        const cbMsg = cbErr instanceof Error ? cbErr.message : String(cbErr);
+                        LogError(`SearchEngine: onProviderResolved callback threw for "${entry.Provider.SourceType}" in scope "${scope.Name}": ${cbMsg}`);
+                    }
+                }
+                return { Source: entry.Provider.SourceType, Results: providerResults };
+            }
+            catch (error) {
+                const msg = error instanceof Error ? error.message : String(error);
+                LogError(`SearchEngine: Provider "${entry.Provider.SourceType}" failed in scope "${scope.Name}": ${msg}`);
+                if (onProviderResolved) {
+                    try {
+                        onProviderResolved({
+                            sourceType: entry.Provider.SourceType,
+                            results: [],
+                            durationMs: Date.now() - providerStart,
+                            scopeID: scope.ID,
+                        });
+                    }
+                    catch { /* swallow */ }
+                }
+                return { Source: entry.Provider.SourceType, Results: [] };
+            }
+        });
+        const labeled = await Promise.all(promises);
+        const sourceCounts = this.countSources(labeled);
+        // Per-scope fusion with weight resolution:
+        //   agent fusion weights > scope.ScopeConfig.fusionWeights > engine defaults
+        const scopeWeights = scopeConfig && typeof scopeConfig.fusionWeights === 'object'
+            ? scopeConfig.fusionWeights
+            : undefined;
+        const fusionWeights = agentFusionWeights ?? scopeWeights;
+        const fused = this._fusion.Fuse(labeled, topK, fusionWeights);
+        return { scopeID: scope.ID, fused, sourceCounts };
+    }
+    /**
+     * Assemble a `ScopeConstraints` for a single scope: Nunjucks-render each template
+     * field against the `SearchContext`, then hand the rendered values to providers.
+     */
+    buildScopeConstraints(bundle, searchContext) {
+        const externalIndexes = bundle.ExternalIndexes.map(row => ({
+            SearchScopeExternalIndexID: row.ID,
+            IndexType: row.IndexType,
+            VectorIndexID: row.VectorIndexID ?? undefined,
+            ExternalIndexName: row.ExternalIndexName ?? undefined,
+            ExternalIndexConfig: this.parseJson(row.ExternalIndexConfig),
+            MetadataFilter: RenderScopeJsonTemplate(row.MetadataFilter, searchContext)
+        }));
+        const entities = bundle.Entities.map(row => ({
+            SearchScopeEntityID: row.ID,
+            EntityID: row.EntityID,
+            EntityName: this.lookupEntityName(row.EntityID),
+            ExtraFilter: row.ExtraFilter ? RenderScopeTemplate(row.ExtraFilter, searchContext) : undefined,
+            UserSearchString: row.UserSearchString ? RenderScopeTemplate(row.UserSearchString, searchContext) : undefined
+        }));
+        const storage = bundle.StorageAccounts.map(row => ({
+            SearchScopeStorageAccountID: row.ID,
+            FileStorageAccountID: row.FileStorageAccountID,
+            FolderPath: row.FolderPath ? RenderScopeTemplate(row.FolderPath, searchContext) : undefined
+        }));
+        // Per-provider query transforms: resolved from SearchScopeProvider.QueryTransformTemplateID
+        // For stored template IDs we need the TemplateEngine — that resolution happens in
+        // Phase 1C (AgentPreExecutionRAG) before this engine is called. We still honor any
+        // pre-rendered transforms that upstream callers placed in the scope config bag.
+        const scopeConfig = this.parseJson(bundle.Scope.ScopeConfig);
+        const rawTransforms = scopeConfig?.perProviderQueryTransforms;
+        const queryTransforms = rawTransforms && typeof rawTransforms === 'object'
+            ? { ...rawTransforms }
+            : undefined;
+        return {
+            ExternalIndexes: externalIndexes.length ? externalIndexes : undefined,
+            Entities: entities.length ? entities : undefined,
+            StorageAccounts: storage.length ? storage : undefined,
+            Context: searchContext,
+            QueryTransforms: queryTransforms,
+            ScopeConfig: scopeConfig ?? undefined
+        };
+    }
+    /** Resolve the EntityID → EntityName via MJ Metadata (for passing to providers that key by name). */
+    lookupEntityName(entityID) {
+        try {
+            const entity = this.ProviderToUse.Entities.find(e => UUIDsEqual(e.ID, entityID));
+            return entity?.Name ?? '';
+        }
+        catch {
+            return '';
+        }
+    }
+    // ────────────────────────────────────────────────────────────────
+    // Re-ranker
+    // ────────────────────────────────────────────────────────────────
+    /**
+     * Pick a re-ranker config for this search. When multiple scopes are in play, we
+     * use the first scope's config (matching task 1B.17: the re-rank stage is one
+     * call applied AFTER cross-scope fusion). A future enhancement could merge
+     * per-scope re-rankers, but the current plan keeps it simple.
+     */
+    pickReRankerConfig(resolvedScopes) {
+        for (const bundle of resolvedScopes) {
+            const scopeConfig = this.parseJson(bundle.Scope.ScopeConfig);
+            const rr = scopeConfig?.reRanker;
+            if (rr?.driverClass)
+                return rr;
+        }
+        return undefined;
+    }
+    /**
+     * Pick the first scope's `RerankerBudgetCents` value to apply to the reranker
+     * run. Mirrors `pickReRankerConfig` — the leading scope's policy wins. NULL
+     * (uncapped) is the default when no scope sets a budget.
+     */
+    pickRerankerBudgetCents(resolvedScopes) {
+        for (const bundle of resolvedScopes) {
+            const cents = bundle.Scope.RerankerBudgetCents;
+            if (cents != null)
+                return cents;
+        }
+        return null;
+    }
+    async runReRanker(query, candidates, cfg, contextUser, budgetGuard) {
+        if (!cfg.driverClass || candidates.length === 0)
+            return candidates;
+        try {
+            const reRanker = MJGlobal.Instance.ClassFactory.CreateInstance(BaseReRanker, cfg.driverClass) ?? new NoopReRanker();
+            const inputTopN = cfg.inputTopN ?? Math.min(100, candidates.length);
+            const outputTopN = cfg.outputTopN ?? Math.min(20, inputTopN);
+            const trimmed = candidates.slice(0, inputTopN);
+            // P2D.6 — pre-call budget short-circuit. When the projected cost would
+            // exceed the remaining budget, skip rerank entirely and return the
+            // unranked top-N. Reported via LogStatus so observability reflects the
+            // skip without surfacing as a failure.
+            if (budgetGuard) {
+                const estimate = reRanker.EstimateCostCents(trimmed.length);
+                if (!budgetGuard.CanSpend(estimate)) {
+                    LogStatus(`SearchEngine: Re-ranker "${cfg.driverClass}" skipped — projected cost ${estimate.toFixed(4)}¢ exceeds remaining budget ${(budgetGuard.Remaining() ?? 0).toFixed(4)}¢ (Spent ${budgetGuard.Spent.toFixed(4)}¢ / Budget ${budgetGuard.Budget ?? 'uncapped'}¢).`);
+                    return trimmed.slice(0, outputTopN);
+                }
+                // Wire post-call cost reporting through the guard so subsequent
+                // EstimateCostCents queries reflect accumulated spend.
+                reRanker.CostReporter = budgetGuard.AsCostReporter();
+            }
+            const ranked = await reRanker.ReRank(query, trimmed, outputTopN, contextUser, cfg.config);
+            LogStatus(`SearchEngine: Re-ranker "${cfg.driverClass}" returned ${ranked.length} result(s) (input=${trimmed.length}, outputTopN=${outputTopN}${budgetGuard ? `, spent=${budgetGuard.Spent.toFixed(4)}¢` : ''})`);
+            return ranked;
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            LogError(`SearchEngine: Re-ranker "${cfg.driverClass}" failed, falling back to unranked: ${msg}`);
+            return candidates;
+        }
+    }
+    // ────────────────────────────────────────────────────────────────
     // Provider loading and initialization
     // ────────────────────────────────────────────────────────────────
     /**
@@ -311,6 +815,7 @@ export class SearchEngine extends BaseSingleton {
                     Priority: record.Priority,
                     SupportsPreview: record.SupportsPreview,
                     MaxResultsOverride: record.MaxResultsOverride ?? null,
+                    Record: record,
                 });
                 LogStatus(`SearchEngine: Provider "${record.Name}" (${driverClass}) enabled`);
             }
@@ -324,13 +829,13 @@ export class SearchEngine extends BaseSingleton {
         }
     }
     // ────────────────────────────────────────────────────────────────
-    // Search execution helpers
+    // Search execution helpers (unscoped path)
     // ────────────────────────────────────────────────────────────────
     /**
      * Run all available providers in parallel and return labeled result lists.
      * When isPreview is true, only providers with SupportsPreview=true are included.
      */
-    async executeProviders(query, topK, filters, contextUser, isPreview) {
+    async executeProviders(query, topK, filters, contextUser, isPreview, scopeConstraints, onProviderResolved) {
         const entries = this._providerEntries.filter(e => {
             if (!e.Provider.IsAvailable())
                 return false;
@@ -343,20 +848,45 @@ export class SearchEngine extends BaseSingleton {
             return [];
         }
         const promises = entries.map(async (entry) => {
+            const providerStart = Date.now();
             try {
                 const providerTopK = entry.MaxResultsOverride ?? topK;
-                const results = await entry.Provider.Search(query, providerTopK, filters, contextUser);
+                const results = await entry.Provider.Search(query, providerTopK, filters, contextUser, scopeConstraints);
                 // Stamp provider metadata onto each result
                 for (const r of results) {
                     r.ProviderId = entry.ID;
                     r.ProviderLabel = entry.DisplayName;
                     r.ProviderIcon = entry.Icon;
                 }
+                if (onProviderResolved) {
+                    try {
+                        onProviderResolved({
+                            sourceType: entry.Provider.SourceType,
+                            results,
+                            durationMs: Date.now() - providerStart,
+                        });
+                    }
+                    catch (cbErr) {
+                        // Streaming callback throwing must NOT corrupt the result; just log.
+                        const cbMsg = cbErr instanceof Error ? cbErr.message : String(cbErr);
+                        LogError(`SearchEngine: onProviderResolved callback threw for "${entry.Provider.SourceType}": ${cbMsg}`);
+                    }
+                }
                 return { Source: entry.Provider.SourceType, Results: results };
             }
             catch (error) {
                 const msg = error instanceof Error ? error.message : String(error);
                 LogError(`SearchEngine: Provider "${entry.Provider.SourceType}" failed: ${msg}`);
+                if (onProviderResolved) {
+                    try {
+                        onProviderResolved({
+                            sourceType: entry.Provider.SourceType,
+                            results: [],
+                            durationMs: Date.now() - providerStart,
+                        });
+                    }
+                    catch { /* swallow */ }
+                }
                 return { Source: entry.Provider.SourceType, Results: [] };
             }
         });
@@ -370,27 +900,32 @@ export class SearchEngine extends BaseSingleton {
         for (const list of lists) {
             switch (list.Source) {
                 case 'vector':
-                    counts.Vector = list.Results.length;
+                    counts.Vector += list.Results.length;
                     break;
                 case 'fulltext':
-                    counts.FullText = list.Results.length;
+                    counts.FullText += list.Results.length;
                     break;
                 case 'entity':
-                    counts.Entity = list.Results.length;
+                    counts.Entity += list.Results.length;
                     break;
                 case 'storage':
-                    counts.Storage = list.Results.length;
+                    counts.Storage += list.Results.length;
                     break;
             }
         }
         return counts;
     }
     // ────────────────────────────────────────────────────────────────
-    // Permission filtering
+    // Permission filtering (residual late safety net)
     // ────────────────────────────────────────────────────────────────
     /**
      * Filter search results by entity-level and row-level security permissions.
      *
+     * **This is a safety net.** Providers are expected to do per-provider permission
+     * push-down (Section 3.6 of plans/search-scopes-rag-plus.md). If this filter is
+     * removing more than a handful of results in practice, the responsible provider's
+     * push-down is incomplete and should be fixed.
+     *
      * Groups results by entity for efficient permission checking:
      * 1. Unknown entities are excluded (fail closed).
      * 2. If the user lacks entity-level CanRead, all results for that entity are dropped.
@@ -411,6 +946,12 @@ export class SearchEngine extends BaseSingleton {
             promises.push(this.filterEntityResults(entityName, groupResults, contextUser, permitted));
         }
         await Promise.all(promises);
+        // Preserve the input order (which is the RRF/re-rank order). groupResultsByEntity
+        // scrambles by entity; re-sort by original position so consumers still see the
+        // best-ranked result first.
+        const inputIndex = new Map();
+        results.forEach((r, i) => inputIndex.set(r, i));
+        permitted.sort((a, b) => (inputIndex.get(a) ?? 0) - (inputIndex.get(b) ?? 0));
         return permitted;
     }
     /**
@@ -507,6 +1048,74 @@ export class SearchEngine extends BaseSingleton {
         }
     }
     /** Build an error SearchResult */
+    /**
+     * Public hook for callers (e.g. the GraphQL resolver) to emit a
+     * Status='Forbidden' SearchExecutionLog row when they reject a request
+     * before delegating to {@link Search}. Without this, forbidden invocations
+     * never reach the analytics dashboard — exactly the signal admins need
+     * to spot users / agents trying to access scopes they shouldn't.
+     */
+    async LogForbiddenSearch(input) {
+        await this.logSearchExecution({
+            Status: 'Forbidden',
+            FailureReason: input.FailureReason,
+            Query: input.Query,
+            ScopeIDs: input.ScopeIDs,
+            StartTime: input.StartTime,
+            ResultCount: 0,
+            RerankerName: null,
+            RerankerCostCents: null,
+            SourceCounts: undefined,
+            ContextUser: input.ContextUser,
+            AIAgentID: input.AIAgentID ?? null,
+        });
+    }
+    /**
+     * Best-effort hook (P3.2) that writes one MJSearchExecutionLog row per
+     * SearchEngine.Search call. Captures query, timing, scope, result count,
+     * reranker info, status, and a per-source-count breakdown for the analytics
+     * dashboard (P3.3) and tuning CSV export (P3.4).
+     *
+     * Errors during the write are swallowed and logged — observability is the
+     * point of this hook, not a load-bearing dependency. A logger that brings
+     * down search would be the worst possible outcome.
+     */
+    async logSearchExecution(input) {
+        try {
+            const log = await this.ProviderToUse.GetEntityObject('MJ: Search Execution Logs', input.ContextUser);
+            log.SearchScopeID = input.ScopeIDs && input.ScopeIDs.length > 0 ? input.ScopeIDs[0] : null;
+            log.UserID = input.ContextUser.ID ?? null;
+            log.AIAgentID = input.AIAgentID ?? null;
+            log.Query = input.Query;
+            log.TotalDurationMs = Date.now() - input.StartTime;
+            log.ResultCount = input.ResultCount;
+            log.RerankerName = input.RerankerName;
+            log.RerankerCostCents = input.RerankerCostCents;
+            log.Status = input.Status;
+            log.FailureReason = input.FailureReason;
+            // ProvidersJSON: per-source breakdown. Per-provider per-call timings
+            // require deeper plumbing through the provider-run loop — deferred to a
+            // later Phase 3 pass. For now, capture the source counts which the
+            // dashboard's hit-rate / volume charts already need.
+            log.ProvidersJSON = input.SourceCounts
+                ? JSON.stringify({
+                    Vector: { ResultCount: input.SourceCounts.Vector },
+                    FullText: { ResultCount: input.SourceCounts.FullText },
+                    Entity: { ResultCount: input.SourceCounts.Entity },
+                    Storage: { ResultCount: input.SourceCounts.Storage },
+                })
+                : null;
+            const saved = await log.Save();
+            if (!saved) {
+                LogError(`SearchEngine: SearchExecutionLog write returned false: ${log.LatestResult?.CompleteMessage ?? 'unknown error'}`);
+            }
+        }
+        catch (err) {
+            // Swallow — this is best-effort observability and must never affect search latency / availability.
+            const msg = err instanceof Error ? err.message : String(err);
+            LogError(`SearchEngine: SearchExecutionLog write threw: ${msg}`);
+        }
+    }
     buildErrorResult(message, startTime) {
         return {
             Success: false,
@@ -529,5 +1138,19 @@ export class SearchEngine extends BaseSingleton {
             Priority: e.Priority,
         }));
     }
+    /** Defensive JSON parse that never throws. Returns `null` on any failure. */
+    parseJson(value) {
+        if (!value)
+            return null;
+        try {
+            const parsed = JSON.parse(value);
+            if (parsed && typeof parsed === 'object')
+                return parsed;
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
 }
 //# sourceMappingURL=SearchEngine.js.map