@memberjunction/search-engine 5.33.0 → 5.34.0

package/dist/generic/SearchEngine.d.ts

@@ -5,7 +5,14 @@
  * fuses results with Reciprocal Rank Fusion (RRF), applies enrichment
  * (entity icons, record names, tags), and filters by minimum score.
  *
- * Providers are loaded via @RegisterClass(BaseSearchProvider, DriverClass) and
+ * When `SearchParams.ScopeIDs` is provided, the engine resolves each scope against
+ * `SearchEngineBase` (which caches all `MJ: Search Scope*` metadata), builds a
+ * `ScopeConstraints` object per scope (including Nunjucks-rendered MetadataFilter,
+ * ExtraFilter, UserSearchString, and FolderPath values), runs each scope's providers
+ * in parallel, then fuses the per-scope results via cross-scope RRF. An optional
+ * re-ranker stage (`BaseReRanker`) runs after fusion when configured.
+ *
+ * Providers are loaded via `@RegisterClass(BaseSearchProvider, DriverClass)` and
  * instantiated using the MJ ClassFactory based on active SearchProvider records.
  *
  * Uses BaseSingleton from @memberjunction/global for a truly global instance.
@@ -15,32 +22,64 @@
 import { IMetadataProvider, UserInfo } from '@memberjunction/core';
 import { SearchEngineBase } from '@memberjunction/core-entities';
 import { BaseSingleton } from '@memberjunction/global';
-import { SearchParams, SearchResult } from './search.types.js';
+import { SearchParams, SearchResult, SearchResultItem, SearchStreamEvent } from './search.types.js';
 /**
  * Configuration options for the SearchEngine.
  */
 export interface SearchEngineConfig {
     /** Default maximum results if not specified in SearchParams (default: 20) */
     DefaultMaxResults?: number;
+    /**
+     * Default multiplier applied to per-provider `topK` to compensate for residual
+     * late permission filtering. Individual calls can override via
+     * `SearchParams.PermissionOverfetchFactor`. Default: 2.
+     */
+    DefaultPermissionOverfetchFactor?: number;
 }
+/**
+ * Callback fired the moment an individual provider's `Search()` resolves —
+ * before fusion, dedup, permission filtering, or rerank. Used by
+ * {@link SearchEngine.streamSearch} to emit `provider` events as each
+ * provider returns rather than waiting for the whole pipeline. The callback
+ * runs inside the provider's promise chain, so any throw it raises will
+ * cancel that provider's contribution but won't take down the search.
+ */
+export type OnProviderResolved = (event: {
+    /** Source type as reported by the provider (e.g. 'vector', 'fulltext'). */
+    sourceType: string;
+    /** Result rows from this provider, with metadata already stamped. */
+    results: SearchResultItem[];
+    /** Wall-clock time spent inside `Provider.Search()` for this invocation. */
+    durationMs: number;
+    /** Scope ID when running per-scope; undefined when unconstrained. */
+    scopeID?: string;
+}) => void;
 /**
  * Singleton search engine that orchestrates multi-source search with RRF fusion.
  *
  * Providers are discovered from the MJ: Search Providers entity. Each active
  * provider's DriverClass is resolved via ClassFactory to create an instance,
- * which is then initialized with the provider's config from the DB.
+ * which is then initialized with the provider's config from the DB record.
  *
  * Usage:
  * ```typescript
  * // Initialize once at server startup
  * await SearchEngine.Instance.Config({}, contextUser);
  *
- * // Execute searches
+ * // Execute searches (unscoped — original behavior)
  * const result = await SearchEngine.Instance.Search({
  *     Query: 'quarterly revenue',
  *     MaxResults: 20,
  *     MinScore: 0.1
  * }, contextUser);
+ *
+ * // Scoped search against two scopes with multi-tenant context
+ * const scopedResult = await SearchEngine.Instance.Search({
+ *     Query: 'refund policy',
+ *     MaxResults: 20,
+ *     ScopeIDs: ['hr-scope-id', 'legal-scope-id'],
+ *     SearchContext: { PrimaryScopeRecordID: 'tenant-a' }
+ * }, contextUser);
  * ```
  */
 export declare class SearchEngine extends BaseSingleton<SearchEngine> {
@@ -52,6 +91,7 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
     private _fusion;
     private _enricher;
     private _defaultMaxResults;
+    private _defaultOverfetchFactor;
     /**
      * Minimum trimmed query length we accept. One- and two-character queries against
      * a `LIKE '%term%'` fan-out are essentially full-database scans with negligible
@@ -87,19 +127,58 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
     /**
      * Execute a multi-source search with RRF fusion and optional enrichment.
      *
-     * Steps:
-     * 1. Run all available providers in parallel
-     * 2. Fuse results with RRF
-     * 3. Deduplicate by EntityName+RecordID
-     * 4. Exclude redundant entity-sourced Content Items
-     * 5. Apply minimum score threshold
-     * 6. Enrich with icons, names, and tags (skipped in preview mode)
+     * When `params.ScopeIDs` is provided, each scope runs independently and the results
+     * are combined via cross-scope RRF before deduplication, re-ranking, and enrichment.
      *
      * @param params - Search parameters
      * @param contextUser - The user performing the search
      * @returns Aggregated search result
      */
     Search(params: SearchParams, contextUser: UserInfo): Promise<SearchResult>;
+    /**
+     * Internal search implementation that optionally fires `onProviderResolved`
+     * as each provider's promise settles. Exposed via the public {@link Search}
+     * (no callback) and {@link streamSearch} (queue-backed callback that
+     * yields `provider` events to the caller).
+     */
+    private searchInternal;
+    /**
+     * Streaming variant of {@link Search}. Yields events as each pipeline
+     * stage produces output so the caller can emit partials to the UI / agent
+     * before fusion + reranking complete.
+     *
+     * **Phase 2C v1 semantics:** runs the same internal pipeline as
+     * {@link Search} and emits synthetic events at each transition. This
+     * preserves all existing fusion / permission / dedup / enrich behavior
+     * — important because those steps have subtle correctness rules that
+     * we don't want to re-implement in a parallel code path. Per-provider
+     * partials are reconstructed from the final SourceCounts; a future
+     * refactor (Phase 2C v2) can split provider emission to true real-time
+     * concurrent emission once we measure that the synthetic phase is the
+     * actual bottleneck.
+     *
+     * Cancellation: the consumer can stop iterating at any point — the
+     * underlying Search() will run to completion but its result is
+     * discarded. AbortSignal-based mid-pipeline cancellation is a Phase 2C
+     * v2 concern.
+     *
+     * Event ordering:
+     *   1. Zero or more `provider` events (one per non-empty source)
+     *   2. Exactly one `fused` event
+     *   3. Optional one `reranked` event (when a reranker is configured)
+     *   4. Exactly one `final` event
+     *   5. On error: a single `error` event in place of `final`.
+     *
+     * @example
+     * for await (const ev of SearchEngine.Instance.streamSearch(params, user)) {
+     *   switch (ev.phase) {
+     *     case 'provider':  scratchpad.append(`${ev.providerName}: ${ev.results.length} hits`); break;
+     *     case 'final':     scratchpad.commit(ev.results); break;
+     *     case 'error':     scratchpad.fail(ev.error); break;
+     *   }
+     * }
+     */
+    streamSearch(params: SearchParams, contextUser: UserInfo): AsyncIterable<SearchStreamEvent>;
     /**
      * Build a stable cache key for a search. Includes the user identity so RLS
      * scopes never bleed across users, plus the trimmed query, MaxResults,
@@ -121,6 +200,36 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
      * @returns Search result in preview mode
      */
     PreviewSearch(query: string, maxResults: number, contextUser: UserInfo): Promise<SearchResult>;
+    /**
+     * Load `ScopeBundle`s for each requested scope ID, filtering out inactive / expired.
+     * Returns an empty array when no scope IDs are supplied (caller treats as Global).
+     */
+    private resolveScopes;
+    /**
+     * Execute all scoped providers for a single scope bundle and return per-scope fused results.
+     */
+    private executeScopeBundle;
+    /**
+     * Assemble a `ScopeConstraints` for a single scope: Nunjucks-render each template
+     * field against the `SearchContext`, then hand the rendered values to providers.
+     */
+    private buildScopeConstraints;
+    /** Resolve the EntityID → EntityName via MJ Metadata (for passing to providers that key by name). */
+    private lookupEntityName;
+    /**
+     * Pick a re-ranker config for this search. When multiple scopes are in play, we
+     * use the first scope's config (matching task 1B.17: the re-rank stage is one
+     * call applied AFTER cross-scope fusion). A future enhancement could merge
+     * per-scope re-rankers, but the current plan keeps it simple.
+     */
+    private pickReRankerConfig;
+    /**
+     * Pick the first scope's `RerankerBudgetCents` value to apply to the reranker
+     * run. Mirrors `pickReRankerConfig` — the leading scope's policy wins. NULL
+     * (uncapped) is the default when no scope sets a budget.
+     */
+    private pickRerankerBudgetCents;
+    private runReRanker;
     /**
      * Instantiate a single provider from its SearchProvider metadata record,
      * initialize it, check availability, and add to the active list if available.
@@ -138,13 +247,18 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
     /**
      * Filter search results by entity-level and row-level security permissions.
      *
+     * **This is a safety net.** Providers are expected to do per-provider permission
+     * push-down (Section 3.6 of plans/search-scopes-rag-plus.md). If this filter is
+     * removing more than a handful of results in practice, the responsible provider's
+     * push-down is incomplete and should be fixed.
+     *
      * Groups results by entity for efficient permission checking:
      * 1. Unknown entities are excluded (fail closed).
      * 2. If the user lacks entity-level CanRead, all results for that entity are dropped.
      * 3. If the user is exempt from RLS, all results pass through.
      * 4. If RLS applies, a RunView validates which record IDs the user can read.
      */
-    private filterByPermissions;
+    protected filterByPermissions(results: SearchResultItem[], contextUser: UserInfo): Promise<SearchResultItem[]>;
     /**
      * Group search result items by EntityName for batch permission checking.
      */
@@ -161,8 +275,36 @@ export declare class SearchEngine extends BaseSingleton<SearchEngine> {
      */
     private filterByRowLevelSecurity;
     /** Build an error SearchResult */
+    /**
+     * Public hook for callers (e.g. the GraphQL resolver) to emit a
+     * Status='Forbidden' SearchExecutionLog row when they reject a request
+     * before delegating to {@link Search}. Without this, forbidden invocations
+     * never reach the analytics dashboard — exactly the signal admins need
+     * to spot users / agents trying to access scopes they shouldn't.
+     */
+    LogForbiddenSearch(input: {
+        Query: string;
+        ScopeIDs?: string[];
+        FailureReason: string;
+        StartTime: number;
+        ContextUser: UserInfo;
+        AIAgentID?: string | null;
+    }): Promise<void>;
+    /**
+     * Best-effort hook (P3.2) that writes one MJSearchExecutionLog row per
+     * SearchEngine.Search call. Captures query, timing, scope, result count,
+     * reranker info, status, and a per-source-count breakdown for the analytics
+     * dashboard (P3.3) and tuning CSV export (P3.4).
+     *
+     * Errors during the write are swallowed and logged — observability is the
+     * point of this hook, not a load-bearing dependency. A logger that brings
+     * down search would be the worst possible outcome.
+     */
+    private logSearchExecution;
     private buildErrorResult;
     /** Build the list of active provider metadata for the response */
     private buildProviderInfoList;
+    /** Defensive JSON parse that never throws. Returns `null` on any failure. */
+    private parseJson;
 }
 //# sourceMappingURL=SearchEngine.d.ts.map

package/dist/generic/SearchEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SearchEngine.d.ts","sourceRoot":"","sources":["../../src/generic/SearchEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAoC,iBAAiB,EAA0C,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC7I,OAAO,EAAE,gBAAgB,EAA0B,MAAM,+BAA+B,CAAC;AACzF,OAAO,EAAE,aAAa,EAA2B,MAAM,wBAAwB,CAAC;AAChF,OAAO,EACH,YAAY,EACZ,YAAY,EAIf,MAAM,gBAAgB,CAAC;AAMxB;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAkBD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,YAAa,SAAQ,aAAa,CAAC,YAAY,CAAC;;IAMzD,wDAAwD;IACxD,WAAkB,QAAQ,IAAI,YAAY,CAEzC;IAED,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,kBAAkB,CAAM;IAEhC;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAK;IAE5C;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAU;IAE9C,wEAAwE;IACxE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAO;IAEhD,OAAO,CAAC,MAAM,CAAqE;IAEnF,gEAAgE;IAChE,SAAS,KAAK,IAAI,IAAI,gBAAgB,CAErC;IAED,8GAA8G;IAC9G,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED;;;;;;;;;;OAUG;IACU,MAAM,CACf,MAAM,EAAE,kBAAuB,EAC/B,WAAW,EAAE,QAAQ,EACrB,YAAY,GAAE,OAAe,GAC9B,OAAO,CAAC,IAAI,CAAC;IAiChB;;;;;;;;;;;;;;OAcG;IACU,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;IAiGvF;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAWrB,6EAA6E;IAC7E,OAAO,CAAC,QAAQ;IAShB,iDAAiD;IAC1C,gBAAgB,IAAI,IAAI;IAI/B;;;;;;;;;OASG;IACU,aAAa,CACtB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAU,EACtB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,YAAY,CAAC;IAYxB;;;OAGG;YACW,kBAAkB;IA4EhC;;;OAGG;YACW,gBAAgB;IAuC9B;;OAEG;IACH,OAAO,CAAC,YAAY;IAyBpB;;;;;;;;OAQG;YACW,mBAAmB;IAyBjC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;OAIG;YACW,mBAAmB;IAiDjC;;;OAGG;YACW,wBAAwB;IA2CtC,kCAAkC;IAClC,OAAO,CAAC,gBAAgB;IAYxB,kEAAkE;IAClE,OAAO,CAAC,qBAAqB;CAUhC"}
+{"version":3,"file":"SearchEngine.d.ts","sourceRoot":"","sources":["../../src/generic/SearchEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAoC,iBAAiB,EAA0C,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC7I,OAAO,EACH,gBAAgB,EAKnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,aAAa,EAAuC,MAAM,wBAAwB,CAAC;AAC5F,OAAO,EACH,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EASpB,MAAM,gBAAgB,CAAC;AAaxB;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,gCAAgC,CAAC,EAAE,MAAM,CAAC;CAC7C;AA4BD;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE;IACrC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB,KAAK,IAAI,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,YAAa,SAAQ,aAAa,CAAC,YAAY,CAAC;;IAMzD,wDAAwD;IACxD,WAAkB,QAAQ,IAAI,YAAY,CAEzC;IAED,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,kBAAkB,CAAM;IAChC,OAAO,CAAC,uBAAuB,CAAK;IAEpC;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAK;IAE5C;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAU;IAE9C,wEAAwE;IACxE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAO;IAEhD,OAAO,CAAC,MAAM,CAAqE;IAEnF,gEAAgE;IAChE,SAAS,KAAK,IAAI,IAAI,gBAAgB,CAErC;IAED,8GAA8G;IAC9G,SAAS,KAAK,aAAa,IAAI,iBAAiB,CAE/C;IAED;;;;;;;;;;OAUG;IACU,MAAM,CACf,MAAM,EAAE,kBAAuB,EAC/B,WAAW,EAAE,QAAQ,EACrB,YAAY,GAAE,OAAe,GAC9B,OAAO,CAAC,IAAI,CAAC;IAkChB;;;;;;;;;OASG;IACU,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC;IAIvF;;;;;OAKG;YACW,cAAc;IAwO5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACW,YAAY,CACtB,MAAM,EAAE,YAAY,EACpB,WAAW,EAAE,QAAQ,GACtB,aAAa,CAAC,iBAAiB,CAAC;IA8GnC;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAWrB,6EAA6E;IAC7E,OAAO,CAAC,QAAQ;IAShB,iDAAiD;IAC1C,gBAAgB,IAAI,IAAI;IAI/B;;;;;;;;;OASG;IACU,aAAa,CACtB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAU,EACtB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,YAAY,CAAC;IAYxB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAerB;;OAEG;YACW,kBAAkB;IAmHhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IA+C7B,qGAAqG;IACrG,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;YAQjB,WAAW;IAgDzB;;;OAGG;YACW,kBAAkB;IA6EhC;;;OAGG;YACW,gBAAgB;IAgE9B;;OAEG;IACH,OAAO,CAAC,YAAY;IAyBpB;;;;;;;;;;;;;OAaG;cACa,mBAAmB,CAC/B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA6B9B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;OAIG;YACW,mBAAmB;IAiDjC;;;OAGG;YACW,wBAAwB;IA2CtC,kCAAkC;IAClC;;;;;;OAMG;IACU,kBAAkB,CAAC,KAAK,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,QAAQ,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBjB;;;;;;;;;OASG;YACW,kBAAkB;IAoDhC,OAAO,CAAC,gBAAgB;IAYxB,kEAAkE;IAClE,OAAO,CAAC,qBAAqB;IAW7B,6EAA6E;IAC7E,OAAO,CAAC,SAAS;CAUpB"}