@memberjunction/core 5.38.0 → 5.40.0

package/dist/generic/providerBase.js

@@ -1,6 +1,7 @@
 import { BaseEntity } from "./baseEntity.js";
-import { EntityDocumentTypeInfo, EntityInfo } from "./entityInfo.js";
+import { EntityDocumentTypeInfo, EntityFieldTSType, EntityInfo, EntityPermissionType } from "./entityInfo.js";
 import { AllMetadata } from "./interfaces.js";
+import { ComputeRRF } from "./scoring/ReciprocalRankFusion.js";
 import { LocalCacheManager } from "./localCacheManager.js";
 import { ApplicationInfo } from "../generic/applicationInfo.js";
 import { AuditLogTypeInfo, AuthorizationInfo, AuthorizationRoleInfo, RoleInfo, RowLevelSecurityFilterInfo, UserInfo } from "./securityInfo.js";
@@ -768,6 +769,255 @@ export class ProviderBase {
         });
         return parts.join('||');
     }
+    /**
+     * Ranked search over **one** entity's records. See {@link IMetadataProvider.SearchEntity}
+     * for the contract and how this differs from {@link EntityByName} /
+     * {@link FullTextSearch}.
+     *
+     * Implementation overview (concrete on `ProviderBase`, used as-is by every
+     * server-side provider; `GraphQLDataProvider` overrides to proxy via GQL):
+     *   1. Resolve the EntityDocument (by `params.options.entityDocumentId`
+     *      override or by looking up the active Search-category doc for the entity).
+     *   2. In parallel: run the lexical pass (RunView with LIKE filters on the
+     *      name field + any `IncludeInUserSearchAPI` fields) and the semantic
+     *      pass (`searchEntitiesSemanticPass`, the protected template method
+     *      each concrete server provider implements).
+     *   3. Fuse via canonical `ComputeRRF()` with optional per-list weights.
+     *   4. Permission-filter via a second RunView constrained to the matched
+     *      record IDs — that pipeline already enforces row-level read perms
+     *      on this entity, so any rows the user can't read drop out.
+     *   5. Slice to topK, apply minScore cutoff, return.
+     */
+    async SearchEntity(params) {
+        const { entityName, searchText } = params;
+        const options = params.options ?? {};
+        const entity = this.EntityByName(entityName);
+        if (!entity) {
+            LogError(`SearchEntity: unknown entity "${entityName}"`);
+            return [];
+        }
+        if (!searchText || !searchText.trim()) {
+            return [];
+        }
+        const mode = options.mode ?? 'hybrid';
+        const topK = options.topK ?? 10;
+        const minScore = options.minScore ?? 0;
+        const overFetch = topK * 2;
+        const contextUser = options.contextUser;
+        // Resolve EntityDocument when semantic ranking is in play
+        let entityDocumentId = null;
+        let embeddingAIModelId = null;
+        if (mode === 'semantic' || mode === 'hybrid') {
+            const resolved = await this.resolveSearchEntityDocument(entity.ID, options.entityDocumentId, contextUser);
+            if (resolved) {
+                entityDocumentId = resolved.id;
+                embeddingAIModelId = resolved.aiModelId;
+            }
+            if (!entityDocumentId && mode === 'semantic') {
+                LogError(`SearchEntity: no active 'Search' EntityDocument for entity "${entityName}"; cannot run semantic-only mode`);
+                return [];
+            }
+        }
+        // Dispatch lexical and semantic in parallel. Wrapping the lexical pass
+        // in Promise.resolve adds no real cost today and keeps the shape ready
+        // if the lexical implementation ever goes async (FTS, external index).
+        const [lexicalRanked, semanticRanked] = await Promise.all([
+            mode === 'semantic'
+                ? Promise.resolve([])
+                : this.searchEntitiesLexicalPass(entity, searchText, overFetch, contextUser),
+            mode === 'lexical' || !entityDocumentId
+                ? Promise.resolve([])
+                : this.searchEntitiesSemanticPass(entityDocumentId, searchText, overFetch, embeddingAIModelId, contextUser),
+        ]);
+        // Blend
+        const blended = this.searchEntitiesBlend(mode, lexicalRanked, semanticRanked, options);
+        // Build component-score lookup for the result-shape step
+        const lexicalScoreById = new Map();
+        for (const c of lexicalRanked)
+            lexicalScoreById.set(c.ID, c.Score);
+        const semanticScoreById = new Map();
+        const semanticDocByRecord = new Map();
+        for (const c of semanticRanked) {
+            semanticScoreById.set(c.ID, c.Score);
+            const erdId = c.Metadata?.['entityRecordDocumentId'];
+            if (erdId)
+                semanticDocByRecord.set(c.ID, erdId);
+        }
+        // Permission-filter post hoc via RunView constrained to the matched
+        // record IDs — the existing pipeline already enforces row-level read
+        // permissions on this entity, so unauthorized rows simply drop out.
+        const ids = blended.map(c => c.ID);
+        const allowedIds = ids.length > 0
+            ? await this.searchEntitiesFilterByPermission(entity, ids, contextUser)
+            : new Set();
+        const results = [];
+        for (const cand of blended) {
+            if (!allowedIds.has(cand.ID))
+                continue;
+            if (cand.Score < minScore)
+                continue;
+            const lex = lexicalScoreById.get(cand.ID);
+            const sem = semanticScoreById.get(cand.ID);
+            const matchType = (lex != null && sem != null)
+                ? 'hybrid'
+                : (lex != null ? 'lexical' : 'semantic');
+            results.push({
+                entityRecordDocumentId: semanticDocByRecord.get(cand.ID) ?? null,
+                recordId: cand.ID,
+                score: cand.Score,
+                matchType,
+                components: { lexical: lex, semantic: sem },
+            });
+            if (results.length >= topK)
+                break;
+        }
+        return results;
+    }
+    /**
+     * Batch form of {@link SearchEntity}. Fans the input list out to N
+     * independent `SearchEntity` calls via `Promise.all`; result arrays come
+     * back aligned by input order (`result[i]` holds the matches for `params[i]`).
+     *
+     * On the server side, the per-entity passes are independent — running them
+     * concurrently is a real wall-clock win when the caller wants results from
+     * multiple entities. On the client side, `GraphQLDataProvider` overrides
+     * this method to pack the whole batch into a single GraphQL round-trip
+     * instead of issuing N parallel HTTP requests.
+     *
+     * See {@link IMetadataProvider.SearchEntities} for the contract.
+     */
+    async SearchEntities(params) {
+        if (!params || params.length === 0)
+            return [];
+        return Promise.all(params.map(p => this.SearchEntity(p)));
+    }
+    /**
+     * Look up the EntityDocument that backs entity search for `entityID`.
+     * Caller supplies an explicit ID override; otherwise we filter for an
+     * Active EntityDocument joined to the 'Search' type via the denormalized
+     * `Type` column on `vwEntityDocuments` (avoids a SQL subquery and keeps
+     * this metadata-layer code provider-agnostic).
+     *
+     * Returns just the EntityDocument PK and its AIModelID; concrete semantic-pass
+     * implementations look up the model from `AIEngine` to recover the driver
+     * class and API name (the view does not project them). Threading AIModelID
+     * through ensures the query embedding is generated with the *same* model
+     * used to build the index — anything else produces garbage cosine scores.
+     */
+    async resolveSearchEntityDocument(entityID, explicitId, contextUser) {
+        const filter = explicitId
+            ? `ID='${explicitId.replace(/'/g, "''")}'`
+            : `EntityID='${entityID.replace(/'/g, "''")}' AND Status='Active' AND Type='Search'`;
+        const r = await this.RunView({
+            EntityName: 'MJ: Entity Documents',
+            ExtraFilter: filter,
+            ResultType: 'simple',
+            MaxRows: 1,
+        }, contextUser);
+        if (!r.Success || (r.Results?.length ?? 0) === 0)
+            return null;
+        const row = r.Results[0];
+        return {
+            id: row.ID,
+            aiModelId: row.AIModelID ?? null,
+        };
+    }
+    /**
+     * Substring/prefix LIKE search against the entity's name field and any
+     * string-typed fields flagged `IncludeInUserSearchAPI`. Returns lexical
+     * scores in [0,1] blended by best match per row.
+     *
+     * **Wildcard handling.** SQL Server's `LIKE` treats `%`, `_`, and `[`
+     * specially; a user searching for `50%_off` would otherwise match far more
+     * than intended. We escape those three characters and declare an explicit
+     * `ESCAPE '\\'` so user-supplied text is matched literally.
+     *
+     * **Field filtering.** Only string-typed fields are searched. Bit/numeric/
+     * date fields can be flagged `IncludeInUserSearchAPI` via metadata edit;
+     * applying `LIKE` to those would error or implicit-convert in subtle ways.
+     */
+    async searchEntitiesLexicalPass(entity, searchText, overFetch, contextUser) {
+        const trimmed = searchText.trim();
+        if (!trimmed)
+            return [];
+        // Escape quotes for SQL string literal, then escape LIKE wildcards
+        // (%, _, [) with the explicit ESCAPE character we declare below.
+        const sanitized = trimmed
+            .replace(/'/g, "''")
+            .replace(/\\/g, '\\\\')
+            .replace(/%/g, '\\%')
+            .replace(/_/g, '\\_')
+            .replace(/\[/g, '\\[');
+        const searchableFields = entity.Fields
+            .filter(f => (f.IncludeInUserSearchAPI || f.IsNameField) && f.TSType === EntityFieldTSType.String);
+        if (searchableFields.length === 0)
+            return [];
+        const likeClauses = searchableFields
+            .map(f => `${f.Name} LIKE '%${sanitized}%' ESCAPE '\\'`)
+            .join(' OR ');
+        const r = await this.RunView({
+            EntityName: entity.Name,
+            ExtraFilter: likeClauses,
+            ResultType: 'simple',
+            MaxRows: overFetch,
+        }, contextUser);
+        if (!r.Success)
+            return [];
+        const lower = trimmed.toLowerCase();
+        const nameField = entity.NameField?.Name ?? entity.Fields.find(f => f.IsNameField)?.Name ?? null;
+        const out = [];
+        for (const row of (r.Results ?? [])) {
+            const id = String(row['ID'] ?? '');
+            if (!id)
+                continue;
+            const nameVal = nameField ? String(row[nameField] ?? '').toLowerCase() : '';
+            let score = 0.5; // any match (in some searchable field)
+            if (nameVal) {
+                if (nameVal === lower)
+                    score = 1.0;
+                else if (nameVal.startsWith(lower))
+                    score = 0.85;
+                else if (nameVal.includes(lower))
+                    score = 0.7;
+            }
+            out.push({ ID: id, Score: score });
+        }
+        // ComputeRRF reads order, not magnitude — sort by score so rank 1 = best lexical match
+        out.sort((a, b) => b.Score - a.Score);
+        return out;
+    }
+    /** Blend lexical + semantic via canonical weighted ComputeRRF. */
+    searchEntitiesBlend(mode, lexicalRanked, semanticRanked, options) {
+        if (mode === 'lexical')
+            return lexicalRanked;
+        if (mode === 'semantic')
+            return semanticRanked;
+        const weights = [
+            options.weights?.lexical ?? 1.0,
+            options.weights?.semantic ?? 1.0,
+        ];
+        return ComputeRRF([lexicalRanked, semanticRanked], options.rrfK ?? 60, weights);
+    }
+    /**
+     * Run a permission-aware RunView restricted to the matched IDs. Whatever
+     * comes back is what the user is allowed to see. Used as a post-filter
+     * over the fused result set.
+     */
+    async searchEntitiesFilterByPermission(entity, ids, contextUser) {
+        if (ids.length === 0)
+            return new Set();
+        const escaped = ids.map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+        const r = await this.RunView({
+            EntityName: entity.Name,
+            ExtraFilter: `ID IN (${escaped})`,
+            Fields: ['ID'],
+            ResultType: 'simple',
+            MaxRows: ids.length,
+        }, contextUser);
+        if (!r.Success)
+            return new Set();
+        return new Set((r.Results ?? []).map(row => row.ID));
+    }
     /**
      * Returns true if any param in the batch has SaveViewResults set,
      * which means the call has a side effect (creating UserViewRun records)
@@ -927,6 +1177,28 @@ export class ProviderBase {
     // ========================================================================
     // PRE-PROCESSING HOOKS
     // ========================================================================
+    /**
+     * Computes the per-user Row-Level-Security WHERE clause that {@link InternalRunView} will
+     * append to this query's SQL for the given user, so it can be folded into the cache
+     * fingerprint. RLS-scoped reads return a different result set than unscoped reads of the same
+     * entity+filter; without including the RLS clause in the cache key, a scoped user could be
+     * served a cached unscoped result set (a data leak).
+     *
+     * Returns '' when the user is exempt from RLS on this entity (the common case), which makes the
+     * resulting fingerprint byte-identical to the pre-RLS format — preserving normal cache sharing.
+     *
+     * Uses `this` (the active provider) to resolve the entity, never the global Metadata, so the
+     * correct per-provider/per-tenant metadata is consulted.
+     */
+    ComputeRunViewRLSWhereClause(params, contextUser) {
+        const user = contextUser ?? this.CurrentUser;
+        if (!user || !params.EntityName)
+            return '';
+        const entity = this.EntityByName(params.EntityName);
+        if (!entity)
+            return '';
+        return entity.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Read, '');
+    }
     /**
      * Pre-processing hook for RunView.
      * Handles telemetry, validation, entity status check, and cache lookup.
@@ -990,7 +1262,8 @@ export class ProviderBase {
         let cachedResult;
         let fingerprint;
         if (willCache && LocalCacheManager.Instance.IsInitialized) {
-            fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
+            const rlsWhereClause = this.ComputeRunViewRLSWhereClause(params, contextUser);
+            fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString, rlsWhereClause);
             const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
             if (cached) {
                 // Filter cached results to only the caller's requested fields (if specified)
@@ -1111,7 +1384,8 @@ export class ProviderBase {
             // BypassCache skips cache entirely — used by maintenance actions querying for
             // records that were inserted via direct SQL (bypassing BaseEntity.Save())
             if (batchWillCache && LocalCacheManager.Instance.IsInitialized) {
-                const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
+                const rlsWhereClause = this.ComputeRunViewRLSWhereClause(param, contextUser);
+                const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString, rlsWhereClause);
                 const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
                 if (cached) {
                     // Filter cached results to caller's requested fields (if specified and not entity_object)
@@ -1517,7 +1791,7 @@ export class ProviderBase {
             // Server-side auto-cache: small, unfiltered, unsorted results are
             // automatically cached even without explicit CacheLocal. These are
             // safe for in-place upsert on entity changes (no filter to evaluate).
-            const fingerprint = preResult.fingerprint || LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
+            const fingerprint = preResult.fingerprint || LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString, this.ComputeRunViewRLSWhereClause(params, contextUser));
             const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
             await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults, result.TotalRowCount, this);
             LogStatusEx({ message: `  📦 [Auto-Cache] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${result.Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
@@ -1560,7 +1834,8 @@ export class ProviderBase {
             if (cacheInfo?.status === 'hit') {
                 continue;
             }
-            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString);
+            const rlsWhereClause = this.ComputeRunViewRLSWhereClause(params[i], contextUser);
+            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString, rlsWhereClause);
             const batchEntityCacheAllowed = this.IsServerCacheAllowedForEntity(params[i]);
             if ((params[i].CacheLocal || this.TrustLocalCacheCompletely) && batchEntityCacheAllowed && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
                 const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);