@memberjunction/search-engine 5.49.0 → 5.50.0

package/dist/generic/SearchEngine.js

@@ -30,6 +30,17 @@ import { BaseReRanker } from './BaseReRanker.js';
 import { NoopReRanker, LoadNoopReRanker } from './NoopReRanker.js';
 import { RerankerBudgetGuard } from '../rerankers/RerankerBudgetGuard.js';
 import { RenderScopeTemplate, RenderScopeJsonTemplate } from './ScopeTemplateRenderer.js';
+import { LaneKindForIndexType } from './ScopeValueEscaper.js';
+import { CheckRenderedTemplate, CheckRequiredMetadataKeys, ParseRequiredMetadataKeys } from './ScopeFilterGuard.js';
+import { ScopeDimensionResolver } from './ScopeDimensionResolver.js';
+import { DefaultSearchScopePermissionResolver } from '../permissions/SearchScopePermissionResolver.js';
+/**
+ * Emitted whenever a scope configures no lanes at all — which in MJ means UNSCOPED, not narrow.
+ * Shared by the dry run and the search path so both produce identical wording.
+ */
+const UNBOUNDED_SCOPE_DIAGNOSTIC = 'this scope configures NO lanes (no external indexes, entities, or storage accounts). ' +
+    'That is not a narrow scope — providers read an empty configuration as UNSCOPED, so it ' +
+    'searches everything available to them with no filter.';
 // Keep the default re-ranker registration alive under tree-shaking
 LoadNoopReRanker();
 /**
@@ -71,18 +82,21 @@ export class SearchEngine extends BaseSingleton {
         this._defaultMaxResults = 20;
         this._defaultOverfetchFactor = 2;
         this._cache = new Map();
+        this._dimensionResolver = new ScopeDimensionResolver();
     }
     /** Static accessor for the global singleton instance */
     static get Instance() {
         return super.getInstance();
     }
     /**
-     * Minimum trimmed query length we accept. One- and two-character queries against
-     * a `LIKE '%term%'` fan-out are essentially full-database scans with negligible
+     * Minimum trimmed query length we accept. A single-character query against a
+     * `LIKE '%term%'` fan-out is essentially a full-database scan with negligible
      * relevance — the providers also enforce this, but we short-circuit here to
-     * avoid the cache lookup and provider dispatch overhead too.
+     * avoid the cache lookup and provider dispatch overhead too. Set to 2 (was 3) so
+     * legitimate short queries aren't silently dropped (bug C3); must stay in lockstep
+     * with the providers' MIN_TERM_LENGTH.
      */
-    static { this.MIN_TERM_LENGTH = 3; }
+    static { this.MIN_TERM_LENGTH = 2; }
     /**
      * Result cache TTL. 30s balances "user resubmits the same prefix" wins against
      * "results stay reasonably fresh after a write". Cache key includes the user's
@@ -91,6 +105,13 @@ export class SearchEngine extends BaseSingleton {
     static { this.CACHE_TTL_MS = 30_000; }
     /** Maximum cached entries across all users. LRU-evicted on overflow. */
     static { this.CACHE_MAX_ENTRIES = 500; }
+    /**
+     * Resolver for a scope's declared Search Context dimensions. Overridable so a host can
+     * supply additional derivation sources (e.g. an external signal) without forking the engine.
+     */
+    get dimensionResolver() {
+        return this._dimensionResolver;
+    }
     /** Access the cached provider metadata from SearchEngineBase */
     get Base() {
         return SearchEngineBase.Instance;
@@ -178,6 +199,8 @@ export class SearchEngine extends BaseSingleton {
                     SourceCounts: undefined,
                     ContextUser: contextUser,
                     AIAgentID: params.AIAgentID ?? null,
+                    AISkillID: params.AISkillID ?? null,
+                    PrimaryScopeRecordID: params.SearchContext?.PrimaryScopeRecordID ?? null,
                 });
                 return this.buildErrorResult('Query cannot be empty', startTime);
             }
@@ -232,6 +255,9 @@ export class SearchEngine extends BaseSingleton {
             // ──────────────────────────────────────────────────────────
             let sourceCounts;
             let fusedResults;
+            // Per-scope decisions, captured for SearchExecutionLog.ScopeDecisionJSON. Empty on
+            // the unconstrained path, which resolves no scope and therefore decides nothing.
+            const scopeDecisions = [];
             if (isUnconstrained) {
                 const labeledLists = await this.executeProviders(params.Query, providerTopK, params.Filters, contextUser, isPreview, undefined, onProviderResolved);
                 sourceCounts = this.countSources(labeledLists);
@@ -240,7 +266,7 @@ export class SearchEngine extends BaseSingleton {
             }
             else {
                 // Run each scope independently, then cross-scope RRF
-                const perScopeRunResults = await Promise.all(resolvedScopes.map(bundle => this.executeScopeBundle(params.Query, providerTopK, params.Filters, contextUser, isPreview, bundle, params.SearchContext, params.FusionWeightsOverride, onProviderResolved)));
+                const perScopeRunResults = await Promise.all(resolvedScopes.map(bundle => this.executeScopeBundle(params.Query, providerTopK, params.Filters, contextUser, isPreview, bundle, params.SearchContext, params.FusionWeightsOverride, this.principalsFrom(params), onProviderResolved)));
                 const sc = { Vector: 0, FullText: 0, Entity: 0, Storage: 0 };
                 const perScopeFused = new Map();
                 for (const r of perScopeRunResults) {
@@ -249,6 +275,7 @@ export class SearchEngine extends BaseSingleton {
                     sc.Entity += r.sourceCounts.Entity;
                     sc.Storage += r.sourceCounts.Storage;
                     perScopeFused.set(r.scopeID, r.fused);
+                    scopeDecisions.push(r.decision);
                 }
                 sourceCounts = sc;
                 if (perScopeFused.size > 1) {
@@ -308,6 +335,9 @@ export class SearchEngine extends BaseSingleton {
                 SourceCounts: sourceCounts,
                 ContextUser: contextUser,
                 AIAgentID: params.AIAgentID ?? null,
+                AISkillID: params.AISkillID ?? null,
+                PrimaryScopeRecordID: params.SearchContext?.PrimaryScopeRecordID ?? null,
+                ScopeDecisions: scopeDecisions,
             });
             const finalResult = {
                 Success: true,
@@ -337,6 +367,8 @@ export class SearchEngine extends BaseSingleton {
                 SourceCounts: undefined,
                 ContextUser: contextUser,
                 AIAgentID: params.AIAgentID ?? null,
+                AISkillID: params.AISkillID ?? null,
+                PrimaryScopeRecordID: params.SearchContext?.PrimaryScopeRecordID ?? null,
             });
             return this.buildErrorResult(msg, startTime);
         }
@@ -483,19 +515,90 @@ export class SearchEngine extends BaseSingleton {
         }
     }
     /**
-     * Build a stable cache key for a search. Includes the user identity so RLS
-     * scopes never bleed across users, plus the trimmed query, MaxResults,
-     * MinScore, and a deterministic projection of Filters.
+     * Deterministically serialize a value with object keys sorted, so that two
+     * logically-identical inputs always produce the same string.
+     *
+     * `JSON.stringify` preserves *insertion* order, which means a caller that builds
+     * `SecondaryScopes` by spreading (a common pattern) can emit the same dimensions in
+     * different orders across calls. Left unsorted that causes avoidable cache misses;
+     * sorted, identity is stable. Array order is PRESERVED — see buildCacheKey for why
+     * `ScopeIDs` order is significant.
+     */
+    stableStringify(value) {
+        if (value === null || typeof value !== 'object')
+            return JSON.stringify(value) ?? 'null';
+        if (Array.isArray(value))
+            return `[${value.map((v) => this.stableStringify(v)).join(',')}]`;
+        const entries = Object.entries(value)
+            .filter(([, v]) => v !== undefined)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+        return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${this.stableStringify(v)}`).join(',')}}`;
+    }
+    /**
+     * Build a stable cache key for a search.
+     *
+     * The key must include EVERY input that can change the result set, or the cache
+     * will serve one caller's results to another. Two of those inputs were previously
+     * missing and both are tenancy/authorization-relevant:
+     *
+     *  - `SearchContext` — carries `PrimaryScopeRecordID` (the TENANT) and the
+     *    `SecondaryScopes` dimensions. Omitting it meant a user with access to two
+     *    tenants could be served the other tenant's results for up to the cache TTL,
+     *    and that two searches differing only by dimension (channel, skill, …)
+     *    collided. This is the reason for the fix.
+     *  - `ScopeIDs` — determines which corpora are searched at all.
+     *
+     * Also folded in: `Mode`, `FusionWeightsOverride` and `PermissionOverfetchFactor`
+     * (all change ranking or the candidate pool, so they change results) and
+     * `AIAgentID` (conservative: agent identity participates in scope resolution and
+     * per-agent overrides upstream; including it can only cost a miss, never leak).
+     *
+     * `ScopeIDs` order is deliberately NOT sorted: it is behaviourally significant,
+     * because cross-scope reranker config and budget are taken from the first scope in
+     * the array that supplies one. Two different orderings can therefore produce
+     * different results and must not share a key.
+     *
+     * The whole projection is emitted through `stableStringify` so key-order variation
+     * in `SecondaryScopes` doesn't fragment the cache. The user ID stays as a readable
+     * prefix for debuggability.
+     *
+     * Note this runs AFTER scope resolution in `searchInternal`, so nothing here is
+     * circular. Entitlement is resolved by the CALLERS (`__Scoped_Search`, the
+     * GraphQL resolvers), which deny before reaching the engine; the engine therefore
+     * never caches across an allow/deny boundary.
      */
     buildCacheKey(trimmed, params, contextUser) {
         const userKey = contextUser?.ID ?? 'anonymous';
         const f = params.Filters ?? {};
-        const filterKey = JSON.stringify({
-            EntityNames: f.EntityNames ? [...f.EntityNames].sort() : undefined,
-            SourceTypes: f.SourceTypes ? [...f.SourceTypes].sort() : undefined,
-            Tags: f.Tags ? [...f.Tags].sort() : undefined,
-        });
-        return `${userKey}|${trimmed}|${params.MaxResults ?? this._defaultMaxResults}|${params.MinScore ?? 0}|${filterKey}`;
+        const projection = {
+            // Order-insensitive: sorted so equivalent filter sets share an entry.
+            Filters: {
+                EntityNames: f.EntityNames ? [...f.EntityNames].sort() : undefined,
+                SourceTypes: f.SourceTypes ? [...f.SourceTypes].sort() : undefined,
+                Tags: f.Tags ? [...f.Tags].sort() : undefined,
+            },
+            MaxResults: params.MaxResults ?? this._defaultMaxResults,
+            MinScore: params.MinScore ?? 0,
+            Mode: params.Mode ?? undefined,
+            // Order-SENSITIVE — do not sort (see doc comment).
+            ScopeIDs: params.ScopeIDs ?? undefined,
+            SearchContext: params.SearchContext
+                ? {
+                    PrimaryScopeEntityID: params.SearchContext.PrimaryScopeEntityID ?? undefined,
+                    PrimaryScopeRecordID: params.SearchContext.PrimaryScopeRecordID ?? undefined,
+                    SecondaryScopes: params.SearchContext.SecondaryScopes ?? undefined,
+                }
+                : undefined,
+            FusionWeightsOverride: params.FusionWeightsOverride ?? undefined,
+            PermissionOverfetchFactor: params.PermissionOverfetchFactor ?? undefined,
+            AIAgentID: params.AIAgentID ?? undefined,
+            // Same reasoning as AIAgentID, and Phase D is what makes it load-bearing: a skill is
+            // a principal that can reach a scope the user's own roles do not grant, and it binds
+            // into expansion queries. Two searches identical but for the active skill are NOT
+            // interchangeable, so they must not share a cache entry.
+            AISkillID: params.AISkillID ?? undefined,
+        };
+        return `${userKey}|${trimmed}|${this.stableStringify(projection)}`;
     }
     /** Insert into the LRU cache, evicting oldest entries when over capacity. */
     cachePut(key, result) {
@@ -529,6 +632,318 @@ export class SearchEngine extends BaseSingleton {
         }, contextUser);
     }
     // ────────────────────────────────────────────────────────────────
+    // Dry run — explain the bound without executing a search
+    // ────────────────────────────────────────────────────────────────
+    /**
+     * Resolve the entire access chain for one or more scopes and report what a search WOULD be
+     * able to reach — **without querying any provider**.
+     *
+     * This is the answer to a question the platform previously could not answer at all: *"as
+     * this user, with this skill active, for this tenant — what is in bounds?"* Every input to
+     * that decision is transient. A grant applies because a time window is open right now; a
+     * dimension is discarded because it was caller-authored on a `ServerDerived` key; a lane is
+     * skipped because its filter lost an `{% if %}` clause. Afterwards, none of it is visible:
+     * a correctly-bounded result set and an accidentally-widened one look identical.
+     *
+     * Note the distinction from {@link PreviewSearch}, which is a real search capped at a few
+     * results. This runs **no** search — it reports the bound, not a sample of what is inside
+     * it. A sample cannot show you an over-broad bound, because the extra documents it would
+     * newly permit are exactly the ones you did not think to look for.
+     *
+     * Two properties make the output trustworthy:
+     *
+     *  - It takes the **same untrusted `SearchContext` a real caller would send**, so the
+     *    preview shows the anti-spoof discard actually happening. A dry run that only accepted
+     *    pre-sanitized input would hide the one thing worth previewing.
+     *  - It reports **every** broken lane in one pass rather than throwing on the first, so a
+     *    misconfigured scope can be fixed in one sitting instead of one error per re-run.
+     *
+     * Unlike a real search this never throws for a scope-level problem; a scope that would fail
+     * closed comes back with `Reachable: false` and the reason, since "it would have failed"
+     * is precisely the finding the caller asked for.
+     *
+     * @param input   scopes to explain plus the hypothetical caller context and principals
+     * @param contextUser the user to evaluate entitlement for
+     * @returns one explanation per requested scope, in the order requested
+     */
+    async ExplainScope(input, contextUser) {
+        const explanations = [];
+        for (const scopeID of input.ScopeIDs) {
+            explanations.push(await this.explainOneScope(scopeID, input, contextUser));
+        }
+        return explanations;
+    }
+    /**
+     * Build the principal set a dimension's expansion query may bind.
+     *
+     * Exists so the real search path and the `ExplainScope` dry run cannot construct principals
+     * differently. They already did once: `ExplainScope` passed the agent and the search path
+     * passed nothing, so any scope deriving its bound from `AgentID` previewed one bound and
+     * searched with another. A single conversion site makes that class of drift unrepresentable
+     * rather than merely fixed.
+     *
+     * Accepts anything carrying the two principal IDs, which both `SearchParams` and
+     * `ExplainScopeInput` do.
+     */
+    principalsFrom(source) {
+        return { AgentID: source.AIAgentID ?? null, SkillID: source.AISkillID ?? null };
+    }
+    /** Explain a single scope. Never throws — a failure to resolve IS the explanation. */
+    async explainOneScope(scopeID, input, contextUser) {
+        const bundle = this.Base.GetScopeBundle(scopeID);
+        const scope = bundle?.Scope ?? this.Base.GetActiveScopeByID(scopeID);
+        if (!bundle || !scope) {
+            return this.buildUnresolvableExplanation(scopeID, input, contextUser);
+        }
+        const entitlement = await this.explainEntitlement(scopeID, input, contextUser);
+        // Resolve dimensions against the caller's UNSANITIZED context, exactly as a real search
+        // would. A ScopeDimensionError means the search would have failed closed — that is a
+        // legitimate result here, so it is reported rather than propagated.
+        let dimensions = [];
+        const diagnostics = [];
+        let effectiveContext;
+        let dimensionFailure = null;
+        try {
+            const resolved = await this.dimensionResolver.Resolve({
+                Scope: scope,
+                CallerContext: input.SearchContext,
+                ContextUser: contextUser,
+                Principals: this.principalsFrom(input),
+            });
+            dimensions = resolved.Provenance;
+            diagnostics.push(...resolved.Diagnostics);
+            effectiveContext = resolved.Context;
+        }
+        catch (e) {
+            dimensionFailure = e instanceof Error ? e.message : String(e);
+            diagnostics.push(`dimension resolution FAILED — a real search would be refused: ${dimensionFailure}`);
+        }
+        // With dimensions unresolved there is no context to render lanes against, so every lane
+        // is reported as skipped for that reason rather than rendered against a partial bound.
+        const lanes = dimensionFailure
+            ? this.buildAllLanesSkipped(bundle, `dimension resolution failed: ${dimensionFailure}`)
+            : this.explainLanes(bundle, effectiveContext);
+        // A scope with NO lanes at all is not a narrow scope — it is MJ's widest. Empty child
+        // collections are collapsed to `undefined` by buildScopeConstraints, and every provider
+        // reads that as "unscoped": all entities, all indexes, no filter. Treating zero lanes as
+        // unreachable inverted the one finding a reviewer most needs, so it is called out
+        // explicitly instead.
+        const hasLanes = lanes.length > 0;
+        if (!hasLanes) {
+            diagnostics.push(UNBOUNDED_SCOPE_DIAGNOSTIC);
+        }
+        const canRetrieve = hasLanes ? lanes.some((l) => l.Status === 'Active') : true;
+        return {
+            ScopeID: scope.ID,
+            ScopeName: scope.Name,
+            Entitlement: entitlement,
+            Dimensions: dimensions,
+            Lanes: lanes,
+            Diagnostics: diagnostics,
+            Reachable: entitlement.Allowed && canRetrieve && !dimensionFailure,
+            Unbounded: !hasLanes,
+            ResolvedContext: effectiveContext,
+        };
+    }
+    /** Resolve entitlement for the dry run, including the skill and tenant principals. */
+    async explainEntitlement(scopeID, input, contextUser) {
+        const principals = {
+            UserID: contextUser.ID ?? null,
+            AgentID: input.AIAgentID ?? null,
+            SkillID: input.AISkillID ?? null,
+            PrimaryScopeRecordID: input.SearchContext?.PrimaryScopeRecordID ?? null,
+        };
+        try {
+            const [agent, skill] = await Promise.all([
+                this.loadPrincipal('MJ: AI Agents', input.AIAgentID, contextUser),
+                this.loadPrincipal('MJ: AI Skills', input.AISkillID, contextUser),
+            ]);
+            const permission = await DefaultSearchScopePermissionResolver.ResolveEffectivePermission({
+                User: contextUser,
+                SearchScopeID: scopeID,
+                Agent: agent,
+                Skill: skill,
+                PrimaryScopeRecordID: principals.PrimaryScopeRecordID,
+                ContextUser: contextUser,
+            });
+            return {
+                Allowed: permission.Allowed,
+                Level: permission.Level,
+                Source: permission.Source,
+                Reason: permission.Reason,
+                Principals: principals,
+            };
+        }
+        catch (e) {
+            // A resolver failure must read as "denied", never as "allowed" — an explanation that
+            // fails open would be worse than no explanation at all.
+            const msg = e instanceof Error ? e.message : String(e);
+            return {
+                Allowed: false,
+                Level: 'None',
+                Source: 'NoGrant',
+                Reason: `entitlement could not be resolved, reported as denied: ${msg}`,
+                Principals: principals,
+            };
+        }
+    }
+    /** Load an agent or skill principal by ID; null when no ID was supplied. */
+    async loadPrincipal(entityName, id, contextUser) {
+        if (!id)
+            return null;
+        const entity = await this.ProviderToUse.GetEntityObject(entityName, contextUser);
+        const loaded = await entity.Load(id);
+        return loaded ? entity : null;
+    }
+    /**
+     * Render every lane and report which would run.
+     *
+     * Reuses `buildScopeConstraints` with a collector rather than duplicating the render logic.
+     * That matters more than it looks: a separate "explain" renderer would be a second
+     * implementation of the guard rules, free to drift from the enforcing one, and a preview
+     * that disagrees with what actually runs is worse than having no preview.
+     */
+    /**
+     * Turn already-built constraints plus a problem map into per-lane explanations.
+     *
+     * Takes the constraints rather than rebuilding them. The previous version called
+     * `buildScopeConstraints` itself, which meant the SEARCH path re-rendered every Nunjucks
+     * template a second time on every scope of every query purely to produce a log record —
+     * pure waste on the hottest path in the engine.
+     *
+     * Both callers still share one rendering pass, which is what keeps the dry run honest:
+     * a separate explain-only renderer would be a second implementation of the guard rules,
+     * free to drift from the enforcing one.
+     */
+    buildLaneExplanations(bundle, constraints, problems) {
+        const lanes = [];
+        for (const row of bundle.ExternalIndexes) {
+            const rendered = constraints.ExternalIndexes?.find((c) => UUIDsEqual(c.SearchScopeExternalIndexID, row.ID));
+            lanes.push({
+                Kind: 'ExternalIndex',
+                Target: row.ExternalIndexName ?? row.ID,
+                LaneID: row.ID,
+                Status: problems.has(row.ID) ? 'Skipped' : 'Active',
+                RenderedFilter: this.stringifyFilter(rendered?.MetadataFilter),
+                RequiredMetadataKeys: this.safeRequiredKeys(row.RequiredMetadataKeys),
+                Reason: problems.get(row.ID),
+            });
+        }
+        for (const row of bundle.Entities) {
+            const rendered = constraints.Entities?.find((c) => UUIDsEqual(c.SearchScopeEntityID, row.ID));
+            lanes.push({
+                Kind: 'Entity',
+                Target: this.lookupEntityName(row.EntityID) || row.EntityID,
+                LaneID: row.ID,
+                Status: problems.has(row.ID) ? 'Skipped' : 'Active',
+                RenderedFilter: rendered?.ExtraFilter ?? null,
+                RequiredMetadataKeys: this.safeRequiredKeys(row.RequiredMetadataKeys),
+                Reason: problems.get(row.ID),
+            });
+        }
+        for (const row of bundle.StorageAccounts) {
+            const rendered = constraints.StorageAccounts?.find((c) => UUIDsEqual(c.SearchScopeStorageAccountID, row.ID));
+            lanes.push({
+                Kind: 'StorageAccount',
+                Target: row.FileStorageAccountID,
+                LaneID: row.ID,
+                // FolderPath is a path prefix, not an access bound, so it is deliberately
+                // unguarded here — consistent with buildScopeConstraints.
+                Status: 'Active',
+                RenderedFilter: rendered?.FolderPath ?? null,
+            });
+        }
+        return lanes;
+    }
+    /** Render every lane for a DRY RUN, collecting problems instead of throwing on the first. */
+    explainLanes(bundle, effectiveContext) {
+        const problems = new Map();
+        try {
+            const constraints = this.buildScopeConstraints(bundle, effectiveContext, problems);
+            return this.buildLaneExplanations(bundle, constraints, problems);
+        }
+        catch (e) {
+            // buildScopeConstraints should not throw with a collector present, but a template
+            // renderer can still fail for reasons the guards do not model.
+            const msg = e instanceof Error ? e.message : String(e);
+            return this.buildAllLanesSkipped(bundle, `constraint building threw: ${msg}`);
+        }
+    }
+    /** Every lane, reported as skipped for one shared reason. */
+    buildAllLanesSkipped(bundle, reason) {
+        return [
+            ...bundle.ExternalIndexes.map((row) => ({
+                Kind: 'ExternalIndex',
+                Target: row.ExternalIndexName ?? row.ID,
+                LaneID: row.ID,
+                Status: 'Skipped',
+                RenderedFilter: null,
+                RequiredMetadataKeys: this.safeRequiredKeys(row.RequiredMetadataKeys),
+                Reason: reason,
+            })),
+            ...bundle.Entities.map((row) => ({
+                Kind: 'Entity',
+                Target: this.lookupEntityName(row.EntityID) || row.EntityID,
+                LaneID: row.ID,
+                Status: 'Skipped',
+                RenderedFilter: null,
+                RequiredMetadataKeys: this.safeRequiredKeys(row.RequiredMetadataKeys),
+                Reason: reason,
+            })),
+            ...bundle.StorageAccounts.map((row) => ({
+                Kind: 'StorageAccount',
+                Target: row.FileStorageAccountID,
+                LaneID: row.ID,
+                Status: 'Skipped',
+                RenderedFilter: null,
+                Reason: reason,
+            })),
+        ];
+    }
+    /** Explanation for a scope that is inactive, missing, or otherwise not loadable. */
+    buildUnresolvableExplanation(scopeID, input, contextUser) {
+        return {
+            ScopeID: scopeID,
+            ScopeName: '(not found)',
+            Entitlement: {
+                Allowed: false,
+                Level: 'None',
+                Source: 'NoGrant',
+                Reason: 'the scope is inactive, expired, or does not exist, so no search can use it',
+                Principals: {
+                    UserID: contextUser.ID ?? null,
+                    AgentID: input.AIAgentID ?? null,
+                    SkillID: input.AISkillID ?? null,
+                    PrimaryScopeRecordID: input.SearchContext?.PrimaryScopeRecordID ?? null,
+                },
+            },
+            Dimensions: [],
+            Lanes: [],
+            Diagnostics: [`scope "${scopeID}" is not an active scope`],
+            Reachable: false,
+            // Not "known to be bounded" — the scope could not be loaded, so nothing about its
+            // configuration was observed. It is unreachable either way.
+            Unbounded: false,
+        };
+    }
+    /** Parse a lane's required-key contract for display; a malformed one reports as empty. */
+    safeRequiredKeys(raw) {
+        try {
+            const keys = ParseRequiredMetadataKeys(raw);
+            return keys.length ? keys : undefined;
+        }
+        catch {
+            // The malformed declaration is already reported as the lane's skip reason.
+            return undefined;
+        }
+    }
+    /** Render a filter of unknown shape as a display string. */
+    stringifyFilter(filter) {
+        if (filter === null || filter === undefined)
+            return null;
+        return typeof filter === 'string' ? filter : JSON.stringify(filter);
+    }
+    // ────────────────────────────────────────────────────────────────
     // Scope resolution
     // ────────────────────────────────────────────────────────────────
     /**
@@ -554,11 +969,58 @@ export class SearchEngine extends BaseSingleton {
     /**
      * Execute all scoped providers for a single scope bundle and return per-scope fused results.
      */
-    async executeScopeBundle(query, topK, filters, contextUser, isPreview, bundle, searchContext, agentFusionWeights, onProviderResolved) {
+    async executeScopeBundle(query, topK, filters, contextUser, isPreview, bundle, searchContext, agentFusionWeights, 
+    /**
+     * Principals available to bind into a dimension's expansion query.
+     *
+     * These MUST match what `ExplainScope` passes. When they did not, a scope whose
+     * `expansionQueryID` binds `AgentID` derived one bound in the dry run and a different
+     * one at search time — making the preview quietly wrong in the only direction anybody
+     * cares about.
+     */
+    principals, onProviderResolved) {
         const scope = bundle.Scope;
         const scopeConfig = this.parseJson(scope.ScopeConfig);
-        const constraints = this.buildScopeConstraints(bundle, searchContext);
+        // Resolve this scope's DECLARED dimensions before building any constraint. For a scope
+        // with no declaration this returns the caller's context untouched (legacy behaviour);
+        // for a declared scope it discards caller-supplied values for ServerDerived keys,
+        // enforces value grammars, applies narrowingOf as a meet, and gives strictValidation
+        // teeth. A ScopeDimensionError propagates so the search fails CLOSED.
+        const dimensionResult = await this.dimensionResolver.Resolve({
+            Scope: bundle.Scope,
+            CallerContext: searchContext,
+            ContextUser: contextUser,
+            Principals: principals,
+        });
+        for (const note of dimensionResult.Diagnostics) {
+            LogStatus(`SearchEngine: scope "${bundle.Scope.Name}" — ${note}`);
+        }
+        const effectiveContext = dimensionResult.Context;
+        const constraints = this.buildScopeConstraints(bundle, effectiveContext);
         const perProviderQueryTransforms = constraints.QueryTransforms ?? {};
+        // Capture the decision for the audit log, REUSING the constraints just built rather
+        // than re-rendering every template a second time. Reaching this line means every lane
+        // guard passed (buildScopeConstraints throws otherwise), so the problem map is empty
+        // and every lane is Active.
+        const unbounded = bundle.ExternalIndexes.length === 0
+            && bundle.Entities.length === 0
+            && bundle.StorageAccounts.length === 0;
+        const decision = {
+            ScopeID: scope.ID,
+            ScopeName: scope.Name,
+            Entitlement: null,
+            Dimensions: dimensionResult.Provenance,
+            Lanes: this.buildLaneExplanations(bundle, constraints, new Map()),
+            // Same warning the dry run emits. The two paths produce one shape, so they should
+            // produce the same prose too — an auditor reading a log row should not have to know
+            // it was written by the search path rather than by a preview.
+            Diagnostics: unbounded
+                ? [...dimensionResult.Diagnostics, UNBOUNDED_SCOPE_DIAGNOSTIC]
+                : dimensionResult.Diagnostics,
+            Reachable: true,
+            Unbounded: unbounded,
+            ResolvedContext: effectiveContext,
+        };
         // Determine which providers this scope participates in (SearchScopeProvider rows)
         const scopeProviderIDs = new Set(bundle.Providers.map(p => NormalizeUUID(p.SearchProviderID)));
         const allowAllProviders = scopeProviderIDs.size === 0; // empty = scope is IsGlobal or all-inclusive
@@ -576,7 +1038,9 @@ export class SearchEngine extends BaseSingleton {
             return {
                 scopeID: scope.ID,
                 fused: [],
-                sourceCounts: { Vector: 0, FullText: 0, Entity: 0, Storage: 0 }
+                sourceCounts: { Vector: 0, FullText: 0, Entity: 0, Storage: 0 },
+                decision: { ...decision, Reachable: false,
+                    Diagnostics: [...decision.Diagnostics, 'no applicable providers for this scope'] },
             };
         }
         // Resolve per-provider `SearchScopeProvider.MaxResultsOverride` if present
@@ -643,32 +1107,122 @@ export class SearchEngine extends BaseSingleton {
             : undefined;
         const fusionWeights = agentFusionWeights ?? scopeWeights;
         const fused = this._fusion.Fuse(labeled, topK, fusionWeights);
-        return { scopeID: scope.ID, fused, sourceCounts };
+        return { scopeID: scope.ID, fused, sourceCounts, decision };
     }
     /**
      * Assemble a `ScopeConstraints` for a single scope: Nunjucks-render each template
      * field against the `SearchContext`, then hand the rendered values to providers.
      */
-    buildScopeConstraints(bundle, searchContext) {
-        const externalIndexes = bundle.ExternalIndexes.map(row => ({
-            SearchScopeExternalIndexID: row.ID,
-            IndexType: row.IndexType,
-            VectorIndexID: row.VectorIndexID ?? undefined,
-            ExternalIndexName: row.ExternalIndexName ?? undefined,
-            ExternalIndexConfig: this.parseJson(RenderScopeTemplate(row.ExternalIndexConfig, searchContext)),
-            MetadataFilter: RenderScopeJsonTemplate(row.MetadataFilter, searchContext)
-        }));
-        const entities = bundle.Entities.map(row => ({
-            SearchScopeEntityID: row.ID,
-            EntityID: row.EntityID,
-            EntityName: this.lookupEntityName(row.EntityID),
-            ExtraFilter: row.ExtraFilter ? RenderScopeTemplate(row.ExtraFilter, searchContext) : undefined,
-            UserSearchString: row.UserSearchString ? RenderScopeTemplate(row.UserSearchString, searchContext) : undefined
-        }));
+    /**
+     * Fail a search CLOSED when a scope field that RESTRICTS was authored but did not render
+     * usably (see `CheckRenderedTemplate`).
+     *
+     * Throwing rather than dropping the offending row is deliberate. Dropping it would empty
+     * the row collection, and `buildScopeConstraints` collapses an empty collection to
+     * `undefined` — which every provider reads as "unscoped", i.e. all entities / all indexes
+     * with no filter. So the surgical-looking fix is the one that widens; failing the search
+     * is the one that doesn't. A broken restricting template is a misconfiguration and should
+     * be loud and actionable, never silently degraded into a wider search.
+     */
+    assertRestrictingTemplateRendered(source, rendered, fieldName, scopeLabel, rowLabel, laneID, collector) {
+        const check = CheckRenderedTemplate(source, rendered);
+        if (check.Status !== 'unusable')
+            return;
+        this.reportLaneProblem(`SearchEngine: scope ${scopeLabel} — ${fieldName} for "${rowLabel}" could not be rendered safely, so the search was NOT run. ${check.Reason}`, laneID, collector);
+    }
+    /**
+     * Enforce a lane's `RequiredMetadataKeys` contract (Phase E).
+     *
+     * The rendered filter must mention every key the author declared. This is the only guard
+     * that catches a filter which rendered *partially* — where an optional `{% if %}` clause
+     * disappeared because its dimension was absent or discarded, leaving a non-empty filter
+     * that passes every other check while restricting on strictly less than intended.
+     */
+    assertRequiredMetadataKeys(declaration, laneID, rendered, scopeLabel, rowLabel, collector) {
+        let requiredKeys;
+        try {
+            requiredKeys = ParseRequiredMetadataKeys(declaration);
+        }
+        catch (e) {
+            // A contract that cannot be parsed must not degrade to "no contract" — that would
+            // turn a typo in the declaration into an unguarded lane.
+            this.reportLaneProblem(`SearchEngine: scope ${scopeLabel} — lane "${rowLabel}" has an unreadable RequiredMetadataKeys declaration, so the lane cannot be trusted. ${e instanceof Error ? e.message : String(e)}`, laneID, collector);
+            return;
+        }
+        if (requiredKeys.length === 0)
+            return;
+        const check = CheckRequiredMetadataKeys(rendered, requiredKeys);
+        if (check.Status !== 'unusable')
+            return;
+        this.reportLaneProblem(`SearchEngine: scope ${scopeLabel} — lane "${rowLabel}" failed its RequiredMetadataKeys contract, so the search was NOT run. ${check.Reason}`, laneID, collector);
+    }
+    /**
+     * Route a lane problem to the right place: throw when enforcing a real search, record when
+     * explaining a hypothetical one.
+     *
+     * A dry run must be able to report *every* broken lane in one pass. If it threw on the first
+     * one, an administrator would fix a scope one error at a time, re-running after each — and
+     * the whole point of the preview is to see the entire picture before anything runs.
+     */
+    reportLaneProblem(message, laneID, collector) {
+        if (collector) {
+            collector.set(laneID, message);
+            LogStatus(message);
+            return;
+        }
+        LogError(message);
+        throw new Error(message);
+    }
+    buildScopeConstraints(bundle, searchContext, collector) {
+        const scopeLabel = `${bundle.Scope.Name} (${bundle.Scope.ID})`;
+        const externalIndexes = bundle.ExternalIndexes.map(row => {
+            const rowLabel = row.ExternalIndexName ?? row.ID;
+            // §5.4: values are escaped for THIS lane's dialect automatically, derived from IndexType.
+            const laneKind = LaneKindForIndexType(row.IndexType);
+            const metadataFilter = RenderScopeJsonTemplate(row.MetadataFilter, searchContext, undefined, laneKind);
+            this.assertRestrictingTemplateRendered(row.MetadataFilter, metadataFilter, 'MetadataFilter', scopeLabel, rowLabel, row.ID, collector);
+            this.assertRequiredMetadataKeys(row.RequiredMetadataKeys, row.ID, metadataFilter, scopeLabel, rowLabel, collector);
+            // ExternalIndexConfig is JSON (namespace/routing), regardless of the filter dialect.
+            const externalIndexConfig = RenderScopeTemplate(row.ExternalIndexConfig, searchContext, undefined, 'json');
+            // ExternalIndexConfig can carry tenant routing (e.g. Pinecone `namespace`), so a
+            // silent render failure here can widen retrieval just like a filter can.
+            this.assertRestrictingTemplateRendered(row.ExternalIndexConfig, externalIndexConfig, 'ExternalIndexConfig', scopeLabel, rowLabel, row.ID, collector);
+            return {
+                SearchScopeExternalIndexID: row.ID,
+                IndexType: row.IndexType,
+                VectorIndexID: row.VectorIndexID ?? undefined,
+                ExternalIndexName: row.ExternalIndexName ?? undefined,
+                ExternalIndexConfig: this.parseJson(externalIndexConfig),
+                MetadataFilter: metadataFilter
+            };
+        });
+        const entities = bundle.Entities.map(row => {
+            // The entity lane is T-SQL, so single quotes must be doubled.
+            const extraFilter = row.ExtraFilter ? RenderScopeTemplate(row.ExtraFilter, searchContext, undefined, 'sql') : undefined;
+            const entityLabel = this.lookupEntityName(row.EntityID) || row.EntityID;
+            this.assertRestrictingTemplateRendered(row.ExtraFilter, extraFilter, 'ExtraFilter', scopeLabel, entityLabel, row.ID, collector);
+            // The SQL lane loses a guarded clause exactly the way an index lane does — same
+            // renderer, same SearchContext, same optional {% if %} blocks — and it is the lane
+            // that reads the operational database, so it gets the same contract.
+            this.assertRequiredMetadataKeys(row.RequiredMetadataKeys, row.ID, extraFilter, scopeLabel, entityLabel, collector);
+            return {
+                SearchScopeEntityID: row.ID,
+                EntityID: row.EntityID,
+                EntityName: this.lookupEntityName(row.EntityID),
+                ExtraFilter: extraFilter,
+                // UserSearchString and FolderPath are NOT restrictions — they shape the query
+                // text / a path prefix — so a soft render there cannot widen an access bound
+                // and is deliberately left unguarded.
+                // 'none' deliberately: this becomes QUERY TEXT, not syntax. Escaping it would corrupt
+                // the search rather than protect it, and it cannot express a bound.
+                UserSearchString: row.UserSearchString ? RenderScopeTemplate(row.UserSearchString, searchContext, undefined, 'none') : undefined
+            };
+        });
         const storage = bundle.StorageAccounts.map(row => ({
             SearchScopeStorageAccountID: row.ID,
             FileStorageAccountID: row.FileStorageAccountID,
-            FolderPath: row.FolderPath ? RenderScopeTemplate(row.FolderPath, searchContext) : undefined
+            // Path traversal, not quoting, is the risk on a storage lane.
+            FolderPath: row.FolderPath ? RenderScopeTemplate(row.FolderPath, searchContext, undefined, 'path') : undefined
         }));
         // Per-provider query transforms: resolved from SearchScopeProvider.QueryTransformTemplateID
         // For stored template IDs we need the TemplateEngine — that resolution happens in
@@ -1062,6 +1616,8 @@ export class SearchEngine extends BaseSingleton {
             SourceCounts: undefined,
             ContextUser: input.ContextUser,
             AIAgentID: input.AIAgentID ?? null,
+            AISkillID: input.AISkillID ?? null,
+            PrimaryScopeRecordID: input.PrimaryScopeRecordID ?? null,
         });
     }
     /**
@@ -1080,6 +1636,15 @@ export class SearchEngine extends BaseSingleton {
             log.SearchScopeID = input.ScopeIDs && input.ScopeIDs.length > 0 ? input.ScopeIDs[0] : null;
             log.UserID = input.ContextUser.ID ?? null;
             log.AIAgentID = input.AIAgentID ?? null;
+            log.AISkillID = input.AISkillID ?? null;
+            log.PrimaryScopeRecordID = input.PrimaryScopeRecordID ?? null;
+            // ScopeDecisionJSON answers "why could this search reach what it reached" — the
+            // dimension provenance and per-lane outcomes, which are otherwise gone the moment
+            // the search returns. Same shape ExplainScope() produces, so a preview taken at
+            // configuration time is directly comparable with what actually ran.
+            log.ScopeDecisionJSON = input.ScopeDecisions?.length
+                ? JSON.stringify(input.ScopeDecisions)
+                : null;
             log.Query = input.Query;
             log.TotalDurationMs = Date.now() - input.StartTime;
             log.ResultCount = input.ResultCount;