@memberjunction/core 5.37.0 → 5.39.0

package/dist/generic/providerBase.js

@@ -1,6 +1,7 @@
 import { BaseEntity } from "./baseEntity.js";
-import { EntityDocumentTypeInfo, EntityInfo } from "./entityInfo.js";
+import { EntityDocumentTypeInfo, EntityFieldTSType, EntityInfo } from "./entityInfo.js";
 import { AllMetadata } from "./interfaces.js";
+import { ComputeRRF } from "./scoring/ReciprocalRankFusion.js";
 import { LocalCacheManager } from "./localCacheManager.js";
 import { ApplicationInfo } from "../generic/applicationInfo.js";
 import { AuditLogTypeInfo, AuthorizationInfo, AuthorizationRoleInfo, RoleInfo, RowLevelSecurityFilterInfo, UserInfo } from "./securityInfo.js";
@@ -768,6 +769,255 @@ export class ProviderBase {
         });
         return parts.join('||');
     }
+    /**
+     * Ranked search over **one** entity's records. See {@link IMetadataProvider.SearchEntity}
+     * for the contract and how this differs from {@link EntityByName} /
+     * {@link FullTextSearch}.
+     *
+     * Implementation overview (concrete on `ProviderBase`, used as-is by every
+     * server-side provider; `GraphQLDataProvider` overrides to proxy via GQL):
+     *   1. Resolve the EntityDocument (by `params.options.entityDocumentId`
+     *      override or by looking up the active Search-category doc for the entity).
+     *   2. In parallel: run the lexical pass (RunView with LIKE filters on the
+     *      name field + any `IncludeInUserSearchAPI` fields) and the semantic
+     *      pass (`searchEntitiesSemanticPass`, the protected template method
+     *      each concrete server provider implements).
+     *   3. Fuse via canonical `ComputeRRF()` with optional per-list weights.
+     *   4. Permission-filter via a second RunView constrained to the matched
+     *      record IDs — that pipeline already enforces row-level read perms
+     *      on this entity, so any rows the user can't read drop out.
+     *   5. Slice to topK, apply minScore cutoff, return.
+     */
+    async SearchEntity(params) {
+        const { entityName, searchText } = params;
+        const options = params.options ?? {};
+        const entity = this.EntityByName(entityName);
+        if (!entity) {
+            LogError(`SearchEntity: unknown entity "${entityName}"`);
+            return [];
+        }
+        if (!searchText || !searchText.trim()) {
+            return [];
+        }
+        const mode = options.mode ?? 'hybrid';
+        const topK = options.topK ?? 10;
+        const minScore = options.minScore ?? 0;
+        const overFetch = topK * 2;
+        const contextUser = options.contextUser;
+        // Resolve EntityDocument when semantic ranking is in play
+        let entityDocumentId = null;
+        let embeddingAIModelId = null;
+        if (mode === 'semantic' || mode === 'hybrid') {
+            const resolved = await this.resolveSearchEntityDocument(entity.ID, options.entityDocumentId, contextUser);
+            if (resolved) {
+                entityDocumentId = resolved.id;
+                embeddingAIModelId = resolved.aiModelId;
+            }
+            if (!entityDocumentId && mode === 'semantic') {
+                LogError(`SearchEntity: no active 'Search' EntityDocument for entity "${entityName}"; cannot run semantic-only mode`);
+                return [];
+            }
+        }
+        // Dispatch lexical and semantic in parallel. Wrapping the lexical pass
+        // in Promise.resolve adds no real cost today and keeps the shape ready
+        // if the lexical implementation ever goes async (FTS, external index).
+        const [lexicalRanked, semanticRanked] = await Promise.all([
+            mode === 'semantic'
+                ? Promise.resolve([])
+                : this.searchEntitiesLexicalPass(entity, searchText, overFetch, contextUser),
+            mode === 'lexical' || !entityDocumentId
+                ? Promise.resolve([])
+                : this.searchEntitiesSemanticPass(entityDocumentId, searchText, overFetch, embeddingAIModelId, contextUser),
+        ]);
+        // Blend
+        const blended = this.searchEntitiesBlend(mode, lexicalRanked, semanticRanked, options);
+        // Build component-score lookup for the result-shape step
+        const lexicalScoreById = new Map();
+        for (const c of lexicalRanked)
+            lexicalScoreById.set(c.ID, c.Score);
+        const semanticScoreById = new Map();
+        const semanticDocByRecord = new Map();
+        for (const c of semanticRanked) {
+            semanticScoreById.set(c.ID, c.Score);
+            const erdId = c.Metadata?.['entityRecordDocumentId'];
+            if (erdId)
+                semanticDocByRecord.set(c.ID, erdId);
+        }
+        // Permission-filter post hoc via RunView constrained to the matched
+        // record IDs — the existing pipeline already enforces row-level read
+        // permissions on this entity, so unauthorized rows simply drop out.
+        const ids = blended.map(c => c.ID);
+        const allowedIds = ids.length > 0
+            ? await this.searchEntitiesFilterByPermission(entity, ids, contextUser)
+            : new Set();
+        const results = [];
+        for (const cand of blended) {
+            if (!allowedIds.has(cand.ID))
+                continue;
+            if (cand.Score < minScore)
+                continue;
+            const lex = lexicalScoreById.get(cand.ID);
+            const sem = semanticScoreById.get(cand.ID);
+            const matchType = (lex != null && sem != null)
+                ? 'hybrid'
+                : (lex != null ? 'lexical' : 'semantic');
+            results.push({
+                entityRecordDocumentId: semanticDocByRecord.get(cand.ID) ?? null,
+                recordId: cand.ID,
+                score: cand.Score,
+                matchType,
+                components: { lexical: lex, semantic: sem },
+            });
+            if (results.length >= topK)
+                break;
+        }
+        return results;
+    }
+    /**
+     * Batch form of {@link SearchEntity}. Fans the input list out to N
+     * independent `SearchEntity` calls via `Promise.all`; result arrays come
+     * back aligned by input order (`result[i]` holds the matches for `params[i]`).
+     *
+     * On the server side, the per-entity passes are independent — running them
+     * concurrently is a real wall-clock win when the caller wants results from
+     * multiple entities. On the client side, `GraphQLDataProvider` overrides
+     * this method to pack the whole batch into a single GraphQL round-trip
+     * instead of issuing N parallel HTTP requests.
+     *
+     * See {@link IMetadataProvider.SearchEntities} for the contract.
+     */
+    async SearchEntities(params) {
+        if (!params || params.length === 0)
+            return [];
+        return Promise.all(params.map(p => this.SearchEntity(p)));
+    }
+    /**
+     * Look up the EntityDocument that backs entity search for `entityID`.
+     * Caller supplies an explicit ID override; otherwise we filter for an
+     * Active EntityDocument joined to the 'Search' type via the denormalized
+     * `Type` column on `vwEntityDocuments` (avoids a SQL subquery and keeps
+     * this metadata-layer code provider-agnostic).
+     *
+     * Returns just the EntityDocument PK and its AIModelID; concrete semantic-pass
+     * implementations look up the model from `AIEngine` to recover the driver
+     * class and API name (the view does not project them). Threading AIModelID
+     * through ensures the query embedding is generated with the *same* model
+     * used to build the index — anything else produces garbage cosine scores.
+     */
+    async resolveSearchEntityDocument(entityID, explicitId, contextUser) {
+        const filter = explicitId
+            ? `ID='${explicitId.replace(/'/g, "''")}'`
+            : `EntityID='${entityID.replace(/'/g, "''")}' AND Status='Active' AND Type='Search'`;
+        const r = await this.RunView({
+            EntityName: 'MJ: Entity Documents',
+            ExtraFilter: filter,
+            ResultType: 'simple',
+            MaxRows: 1,
+        }, contextUser);
+        if (!r.Success || (r.Results?.length ?? 0) === 0)
+            return null;
+        const row = r.Results[0];
+        return {
+            id: row.ID,
+            aiModelId: row.AIModelID ?? null,
+        };
+    }
+    /**
+     * Substring/prefix LIKE search against the entity's name field and any
+     * string-typed fields flagged `IncludeInUserSearchAPI`. Returns lexical
+     * scores in [0,1] blended by best match per row.
+     *
+     * **Wildcard handling.** SQL Server's `LIKE` treats `%`, `_`, and `[`
+     * specially; a user searching for `50%_off` would otherwise match far more
+     * than intended. We escape those three characters and declare an explicit
+     * `ESCAPE '\\'` so user-supplied text is matched literally.
+     *
+     * **Field filtering.** Only string-typed fields are searched. Bit/numeric/
+     * date fields can be flagged `IncludeInUserSearchAPI` via metadata edit;
+     * applying `LIKE` to those would error or implicit-convert in subtle ways.
+     */
+    async searchEntitiesLexicalPass(entity, searchText, overFetch, contextUser) {
+        const trimmed = searchText.trim();
+        if (!trimmed)
+            return [];
+        // Escape quotes for SQL string literal, then escape LIKE wildcards
+        // (%, _, [) with the explicit ESCAPE character we declare below.
+        const sanitized = trimmed
+            .replace(/'/g, "''")
+            .replace(/\\/g, '\\\\')
+            .replace(/%/g, '\\%')
+            .replace(/_/g, '\\_')
+            .replace(/\[/g, '\\[');
+        const searchableFields = entity.Fields
+            .filter(f => (f.IncludeInUserSearchAPI || f.IsNameField) && f.TSType === EntityFieldTSType.String);
+        if (searchableFields.length === 0)
+            return [];
+        const likeClauses = searchableFields
+            .map(f => `${f.Name} LIKE '%${sanitized}%' ESCAPE '\\'`)
+            .join(' OR ');
+        const r = await this.RunView({
+            EntityName: entity.Name,
+            ExtraFilter: likeClauses,
+            ResultType: 'simple',
+            MaxRows: overFetch,
+        }, contextUser);
+        if (!r.Success)
+            return [];
+        const lower = trimmed.toLowerCase();
+        const nameField = entity.NameField?.Name ?? entity.Fields.find(f => f.IsNameField)?.Name ?? null;
+        const out = [];
+        for (const row of (r.Results ?? [])) {
+            const id = String(row['ID'] ?? '');
+            if (!id)
+                continue;
+            const nameVal = nameField ? String(row[nameField] ?? '').toLowerCase() : '';
+            let score = 0.5; // any match (in some searchable field)
+            if (nameVal) {
+                if (nameVal === lower)
+                    score = 1.0;
+                else if (nameVal.startsWith(lower))
+                    score = 0.85;
+                else if (nameVal.includes(lower))
+                    score = 0.7;
+            }
+            out.push({ ID: id, Score: score });
+        }
+        // ComputeRRF reads order, not magnitude — sort by score so rank 1 = best lexical match
+        out.sort((a, b) => b.Score - a.Score);
+        return out;
+    }
+    /** Blend lexical + semantic via canonical weighted ComputeRRF. */
+    searchEntitiesBlend(mode, lexicalRanked, semanticRanked, options) {
+        if (mode === 'lexical')
+            return lexicalRanked;
+        if (mode === 'semantic')
+            return semanticRanked;
+        const weights = [
+            options.weights?.lexical ?? 1.0,
+            options.weights?.semantic ?? 1.0,
+        ];
+        return ComputeRRF([lexicalRanked, semanticRanked], options.rrfK ?? 60, weights);
+    }
+    /**
+     * Run a permission-aware RunView restricted to the matched IDs. Whatever
+     * comes back is what the user is allowed to see. Used as a post-filter
+     * over the fused result set.
+     */
+    async searchEntitiesFilterByPermission(entity, ids, contextUser) {
+        if (ids.length === 0)
+            return new Set();
+        const escaped = ids.map(id => `'${id.replace(/'/g, "''")}'`).join(',');
+        const r = await this.RunView({
+            EntityName: entity.Name,
+            ExtraFilter: `ID IN (${escaped})`,
+            Fields: ['ID'],
+            ResultType: 'simple',
+            MaxRows: ids.length,
+        }, contextUser);
+        if (!r.Success)
+            return new Set();
+        return new Set((r.Results ?? []).map(row => row.ID));
+    }
     /**
      * Returns true if any param in the batch has SaveViewResults set,
      * which means the call has a side effect (creating UserViewRun records)
@@ -1273,9 +1523,16 @@ export class ProviderBase {
         const preResolvedCache = currentFingerprints.length > 0
             ? await LocalCacheManager.Instance.GetRunViewResults(currentFingerprints)
             : new Map();
+        // Pre-build index Map for server results lookup — avoids O(N^2) .find() scans below.
+        const serverResultsByViewIndex = new Map();
+        if (response.results) {
+            for (const r of response.results) {
+                serverResultsByViewIndex.set(r.viewIndex, r);
+            }
+        }
         // Process all results in parallel — 'current' entries now read from the
         // pre-resolved map instead of issuing their own GetRunViewResult calls.
-        const processingPromises = params.map((param, i) => this.processSingleSmartCacheResult(param, i, response.results, preResolvedCache, contextUser));
+        const processingPromises = params.map((param, i) => this.processSingleSmartCacheResult(param, i, serverResultsByViewIndex.get(i), preResolvedCache, contextUser));
         const processedResults = await Promise.all(processingPromises);
         // Aggregate telemetry stats
         let cacheHits = 0;
@@ -1305,8 +1562,7 @@ export class ProviderBase {
      *   'current' items hit this map instead of issuing per-param IDB reads,
      *   amortizing IndexedDB transaction overhead across the whole batch.
      */
-    async processSingleSmartCacheResult(param, index, serverResults, preResolvedCache, contextUser) {
-        const checkResult = serverResults.find(r => r.viewIndex === index);
+    async processSingleSmartCacheResult(param, index, checkResult, preResolvedCache, contextUser) {
         if (!checkResult) {
             return {
                 result: {
@@ -2204,30 +2460,59 @@ export class ProviderBase {
      * @param settings - Array of entity settings metadata
      * @returns Processed array of EntityInfo instances with all relationships established
      */
+    /**
+     * Groups items into a Map keyed by a NormalizeUUID-transformed value.
+     * Single-pass O(N) helper used to build pre-indexed lookup maps for
+     * metadata post-processing, replacing repeated O(N*M) filter scans.
+     */
+    groupByNormalizedUUID(items, keyFn) {
+        const map = new Map();
+        for (const item of items) {
+            const key = NormalizeUUID(keyFn(item));
+            let list = map.get(key);
+            if (!list) {
+                list = [];
+                map.set(key, list);
+            }
+            list.push(item);
+        }
+        return map;
+    }
     PostProcessEntityMetadata(entities, fields, fieldValues, permissions, relationships, settings, organicKeys, organicKeyRelatedEntities) {
         const result = [];
         // Sort entities alphabetically by name to ensure deterministic ordering
         // This prevents non-deterministic output in CodeGen and other metadata consumers
         const sortedEntities = entities.sort((a, b) => a.Name.localeCompare(b.Name));
-        if (fieldValues && fieldValues.length > 0)
-            for (let f of fields) {
-                // populate the field values for each field, if we have them
-                f.EntityFieldValues = fieldValues.filter(fv => UUIDsEqual(fv.EntityFieldID, f.ID));
+        if (fieldValues && fieldValues.length > 0) {
+            const fieldValuesByFieldId = this.groupByNormalizedUUID(fieldValues, fv => fv.EntityFieldID);
+            for (const f of fields) {
+                f.EntityFieldValues = fieldValuesByFieldId.get(NormalizeUUID(f.ID)) || [];
             }
+        }
         // Link organic key related entities to their parent organic keys
         if (organicKeys && organicKeyRelatedEntities && organicKeyRelatedEntities.length > 0) {
+            const okreByOrganicKeyId = this.groupByNormalizedUUID(organicKeyRelatedEntities, okre => okre.EntityOrganicKeyID);
             for (const ok of organicKeys) {
-                ok.EntityOrganicKeyRelatedEntities = organicKeyRelatedEntities.filter(okre => UUIDsEqual(okre.EntityOrganicKeyID, ok.ID));
+                ok.EntityOrganicKeyRelatedEntities = okreByOrganicKeyId.get(NormalizeUUID(ok.ID)) || [];
             }
         }
-        for (let e of sortedEntities) {
-            e.EntityFields = fields.filter(f => UUIDsEqual(f.EntityID, e.ID)).sort((a, b) => a.Sequence - b.Sequence);
-            e.EntityPermissions = permissions.filter(p => UUIDsEqual(p.EntityID, e.ID));
-            e.EntityRelationships = relationships.filter(r => UUIDsEqual(r.EntityID, e.ID));
-            e.EntitySettings = settings.filter(s => UUIDsEqual(s.EntityID, e.ID));
+        const fieldsByEntityId = this.groupByNormalizedUUID(fields, f => f.EntityID);
+        const permissionsByEntityId = this.groupByNormalizedUUID(permissions, p => p.EntityID);
+        const relationshipsByEntityId = this.groupByNormalizedUUID(relationships, r => r.EntityID);
+        const settingsByEntityId = this.groupByNormalizedUUID(settings, s => s.EntityID);
+        const activeOrganicKeysByEntityId = organicKeys
+            ? this.groupByNormalizedUUID(organicKeys.filter(ok => ok.Status === 'Active'), ok => ok.EntityID)
+            : new Map();
+        for (const e of sortedEntities) {
+            const entityIdKey = NormalizeUUID(e.ID);
+            const entityFields = fieldsByEntityId.get(entityIdKey) || [];
+            e.EntityFields = entityFields.sort((a, b) => a.Sequence - b.Sequence);
+            e.EntityPermissions = permissionsByEntityId.get(entityIdKey) || [];
+            e.EntityRelationships = relationshipsByEntityId.get(entityIdKey) || [];
+            e.EntitySettings = settingsByEntityId.get(entityIdKey) || [];
             // Link active organic keys to the entity
             if (organicKeys) {
-                e.EntityOrganicKeys = organicKeys.filter(ok => UUIDsEqual(ok.EntityID, e.ID) && ok.Status === 'Active');
+                e.EntityOrganicKeys = activeOrganicKeysByEntityId.get(entityIdKey) || [];
             }
             result.push(new EntityInfo(e));
         }
@@ -2330,66 +2615,39 @@ export class ProviderBase {
     get AuthorizationRoles() {
         return this._localMetadata.AllAuthorizationRoles;
     }
-    /**
-     * Gets all saved queries in the system.
-     * @returns Array of QueryInfo objects representing stored queries
-     */
+    /** @deprecated Use `QueryEngine.Instance.Queries` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get Queries() {
         return this._localMetadata.AllQueries;
     }
-    /**
-     * Gets all query category definitions.
-     * @returns Array of QueryCategoryInfo objects for query organization
-     */
+    /** @deprecated Use `QueryEngine.Instance.Categories` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryCategories() {
         return this._localMetadata.AllQueryCategories;
     }
-    /**
-     * Gets all query field definitions.
-     * @returns Array of QueryFieldInfo objects defining query result columns
-     */
+    /** @deprecated Use `QueryEngine.Instance.Fields` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryFields() {
         return this._localMetadata.AllQueryFields;
     }
-    /**
-     * Gets all query permission assignments.
-     * @returns Array of QueryPermissionInfo objects defining query access
-     */
+    /** @deprecated Use `QueryEngine.Instance.Permissions` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryPermissions() {
         return this._localMetadata.AllQueryPermissions;
     }
-    /**
-     * Gets all query entity associations.
-     * @returns Array of QueryEntityInfo objects linking queries to entities
-     */
+    /** @deprecated Use `QueryEngine.Instance.QueryEntities` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryEntities() {
         return this._localMetadata.AllQueryEntities;
     }
-    /**
-     * Gets all query parameter definitions.
-     * @returns Array of QueryParameterInfo objects for parameterized queries
-     */
+    /** @deprecated Use `QueryEngine.Instance.Parameters` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryParameters() {
         return this._localMetadata.AllQueryParameters;
     }
-    /**
-     * Gets all query dependency records tracking composition references between queries.
-     * @returns Array of QueryDependencyInfo objects representing query-to-query dependencies
-     */
+    /** @deprecated Use `QueryEngine.Instance.Dependencies` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QueryDependencies() {
         return this._localMetadata.AllQueryDependencies;
     }
-    /**
-     * Gets all SQL dialect definitions.
-     * @returns Array of SQLDialectInfo objects representing supported SQL dialects
-     */
+    /** @deprecated Use `QueryEngine.Instance.SQLDialects` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get SQLDialects() {
         return this._localMetadata.AllSQLDialects;
     }
-    /**
-     * Gets all query SQL dialect variants.
-     * @returns Array of QuerySQLInfo objects containing dialect-specific SQL for queries
-     */
+    /** @deprecated Use `QueryEngine.Instance.QuerySQLs` from `@memberjunction/core-entities`. Will be removed in v6.x. */
     get QuerySQLs() {
         return this._localMetadata.AllQuerySQLs;
     }