@memberjunction/core 5.21.0 → 5.23.0

package/dist/generic/providerBase.js

@@ -15,6 +15,7 @@ import { Metadata } from "./metadata.js";
 import { RunView } from "../views/runView.js";
 import { IsPlatformSQL } from "./platformSQL.js";
 import { GetDataHooks } from "./dataHooks.js";
+import { TransformSimpleObjectToEntityObject } from "./util.js";
 /**
  * Creates a new instance of AllMetadata from a simple object.
  * Handles deserialization and proper instantiation of all metadata classes.
@@ -529,10 +530,12 @@ export class ProviderBase {
             boundaries.push({ start: allParams.length, count: entry.params.length });
             allParams.push(...entry.params);
         }
-        const entityNames = allParams.map(p => p.EntityName || p.ViewName || '?').join(', ');
-        LogStatusEx({
-            message: `⚡ [Coalesce] Merged ${queue.length} RunViews calls (${allParams.length} total entities) into 1 mega-batch: [${entityNames}]`,
-            verboseOnly: false
+        const entityNames = allParams.map(p => p.EntityName || p.ViewName || '?');
+        const eventId = TelemetryManager.Instance.StartEvent('Coalesce', 'ProviderBase.flushCoalesceQueue', {
+            CallerCount: queue.length,
+            TotalEntityCount: allParams.length,
+            Entities: entityNames,
+            CallerBoundaries: [...boundaries]
         });
         try {
             // Execute the mega-batch as a single pipeline call
@@ -543,8 +546,10 @@ export class ProviderBase {
                 const callerResults = allResults.slice(start, start + count);
                 queue[i].resolve(callerResults);
             }
+            TelemetryManager.Instance.EndEvent(eventId);
         }
         catch (err) {
+            TelemetryManager.Instance.EndEvent(eventId, { success: false, error: String(err) });
             // If the mega-batch fails, reject all callers
             for (const entry of queue) {
                 entry.reject(err);
@@ -555,6 +560,111 @@ export class ProviderBase {
      * Runs RunViews without coalescing (used internally after coalescing merge).
      * This goes through dedup + linger + the full pipeline.
      */
+    /**
+     * Performs a full-text search across all entities that have FullTextSearchEnabled=true.
+     * Uses the existing RunView + UserSearchString infrastructure which routes through
+     * the database-native FTS capabilities (SQL Server FREETEXT functions, PostgreSQL tsvector).
+     *
+     * This is the default implementation that works across all database providers. Each provider's
+     * createViewUserSearchSQL() method handles the platform-specific SQL generation.
+     *
+     * @see /packages/MJCore/docs/FULL_TEXT_SEARCH_GUIDE.md for comprehensive documentation
+     */
+    async FullTextSearch(params, contextUser) {
+        const startTime = Date.now();
+        try {
+            const md = new Metadata();
+            const maxRows = params.MaxRowsPerEntity ?? 10;
+            // Get all FTS-enabled entities
+            const ftsEntities = this.resolveFTSEntities(md, params.EntityNames);
+            if (ftsEntities.length === 0) {
+                return {
+                    Success: true,
+                    Results: [],
+                    TotalCount: 0,
+                    EntitiesSearched: 0,
+                    ElapsedMs: Date.now() - startTime
+                };
+            }
+            // Build RunView params for each FTS entity
+            const viewParams = ftsEntities.map(entity => ({
+                EntityName: entity.Name,
+                UserSearchString: params.SearchText,
+                MaxRows: maxRows,
+                ResultType: 'simple'
+            }));
+            // Execute all searches in parallel via RunViews
+            const viewResults = await this.RunViews(viewParams, contextUser);
+            // Convert results to FullTextSearchResultItems
+            const allResults = [];
+            for (let i = 0; i < viewResults.length; i++) {
+                const result = viewResults[i];
+                const entity = ftsEntities[i];
+                if (!result.Success)
+                    continue;
+                const titleField = this.findBestField(entity, ['Name', 'Title', 'Subject', 'Label']);
+                const snippetField = this.findBestField(entity, ['Description', 'Summary', 'Body', 'Content', 'Text', 'Notes']);
+                for (let j = 0; j < result.Results.length; j++) {
+                    const record = result.Results[j];
+                    allResults.push({
+                        EntityName: entity.Name,
+                        RecordID: String(record[entity.FirstPrimaryKey?.Name ?? 'ID'] ?? ''),
+                        Title: String(record[titleField] ?? 'Untitled'),
+                        Snippet: String(record[snippetField] ?? '').substring(0, 200),
+                        Score: 1.0 / (j + 1) // Rank-based scoring for RRF compatibility
+                    });
+                }
+            }
+            // Sort by score descending
+            allResults.sort((a, b) => b.Score - a.Score);
+            return {
+                Success: true,
+                Results: allResults,
+                TotalCount: allResults.length,
+                EntitiesSearched: ftsEntities.length,
+                ElapsedMs: Date.now() - startTime
+            };
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            LogError(`FullTextSearch failed: ${msg}`);
+            return {
+                Success: false,
+                ErrorMessage: msg,
+                Results: [],
+                TotalCount: 0,
+                EntitiesSearched: 0,
+                ElapsedMs: Date.now() - startTime
+            };
+        }
+    }
+    /**
+     * Resolve which entities to search. Filters to only FTS-enabled entities.
+     * If entityNames provided, intersects with the FTS-enabled set.
+     */
+    resolveFTSEntities(md, entityNames) {
+        const allEntities = md.Entities.filter(e => e.FullTextSearchEnabled);
+        if (!entityNames || entityNames.length === 0) {
+            return allEntities;
+        }
+        // Intersect requested names with FTS-enabled entities only
+        const nameSet = new Set(entityNames.map(n => n.toLowerCase()));
+        return allEntities.filter(e => nameSet.has(e.Name.toLowerCase()));
+    }
+    /**
+     * Find the best display field from an entity by checking preferred field names.
+     * Returns the first match or falls back to the first text field.
+     */
+    findBestField(entity, preferredNames) {
+        for (const name of preferredNames) {
+            const field = entity.Fields.find(f => f.Name === name);
+            if (field)
+                return field.Name;
+        }
+        // Fallback to first text field
+        const textField = entity.Fields.find(f => f.Type.toLowerCase().includes('varchar') || f.Type.toLowerCase().includes('text'));
+        return textField?.Name ?? entity.FirstPrimaryKey?.Name ?? 'ID';
+    }
     async RunViewsUncoalesced(params, contextUser) {
         const key = this.GenerateDedupKey(params, contextUser);
         const existing = this._inflightViews.get(key);
@@ -828,7 +938,7 @@ export class ProviderBase {
                     Success: true,
                     Results: cached.results,
                     RowCount: cached.results.length,
-                    TotalRowCount: cached.results.length,
+                    TotalRowCount: cached.totalRowCount ?? cached.results.length,
                     ExecutionTime: 0, // Cached, no execution time
                     ErrorMessage: '',
                     UserViewRunID: '',
@@ -956,7 +1066,7 @@ export class ProviderBase {
                         Success: true,
                         Results: cached.results,
                         RowCount: cached.results.length,
-                        TotalRowCount: cached.results.length,
+                        TotalRowCount: cached.totalRowCount ?? cached.results.length,
                         ExecutionTime: 0,
                         ErrorMessage: '',
                         UserViewRunID: '',
@@ -1119,7 +1229,7 @@ export class ProviderBase {
                     Success: true,
                     Results: cached.results,
                     RowCount: cached.rowCount,
-                    TotalRowCount: cached.rowCount,
+                    TotalRowCount: cached.totalRowCount ?? cached.rowCount,
                     ExecutionTime: 0,
                     ErrorMessage: '',
                     UserViewRunID: '',
@@ -1161,7 +1271,7 @@ export class ProviderBase {
                         Success: true,
                         Results: merged.results,
                         RowCount: merged.rowCount,
-                        TotalRowCount: merged.rowCount,
+                        TotalRowCount: merged.totalRowCount ?? merged.rowCount,
                         ExecutionTime: 0,
                         ErrorMessage: '',
                         UserViewRunID: '',
@@ -1182,11 +1292,12 @@ export class ProviderBase {
         }
         else if (checkResult.status === 'stale') {
             // Cache is stale - use fresh data and update cache (entity doesn't support differential)
+            const staleResults = checkResult.results || [];
             const freshResult = {
                 Success: true,
-                Results: checkResult.results || [],
-                RowCount: checkResult.rowCount || 0,
-                TotalRowCount: checkResult.rowCount || 0,
+                Results: staleResults,
+                RowCount: staleResults.length,
+                TotalRowCount: checkResult.rowCount || staleResults.length,
                 ExecutionTime: 0,
                 ErrorMessage: '',
                 UserViewRunID: '',
@@ -1197,8 +1308,8 @@ export class ProviderBase {
                 const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
                 // Note: We don't await here to avoid blocking the response
                 // Cache update happens in background
-                LocalCacheManager.Instance.SetRunViewResult(fingerprint, param, checkResult.results || [], checkResult.maxUpdatedAt, checkResult.aggregateResults // Include aggregate results in cache
-                ).catch(e => LogError(`Failed to update cache: ${e}`));
+                LocalCacheManager.Instance.SetRunViewResult(fingerprint, param, checkResult.results || [], checkResult.maxUpdatedAt, checkResult.aggregateResults, // Include aggregate results in cache
+                checkResult.rowCount).catch(e => LogError(`Failed to update cache: ${e}`));
             }
             // Transform to entity objects if needed
             await this.TransformSimpleObjectToEntityObject(param, freshResult, contextUser);
@@ -1287,9 +1398,9 @@ export class ProviderBase {
         // with circular subscriber references that break JSON.stringify.
         // On cache read, TransformSimpleObjectToEntityObject is called to restore
         // entity objects when ResultType === 'entity_object'.
-        if (params.CacheLocal && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
+        if ((params.CacheLocal || this.TrustLocalCacheCompletely) && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
             const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
-            await LocalCacheManager.Instance.SetRunViewResult(preResult.fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults);
+            await LocalCacheManager.Instance.SetRunViewResult(preResult.fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults, result.TotalRowCount);
         }
         else if (this.shouldAutoCache(params, result)) {
             // Server-side auto-cache: small, unfiltered, unsorted results are
@@ -1297,7 +1408,7 @@ export class ProviderBase {
             // safe for in-place upsert on entity changes (no filter to evaluate).
             const fingerprint = preResult.fingerprint || LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
             const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
-            await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults);
+            await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults, result.TotalRowCount);
             LogStatusEx({ message: `  📦 [Auto-Cache] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${result.Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
         }
         // Transform the result set into BaseEntity-derived objects, if needed
@@ -1339,13 +1450,13 @@ export class ProviderBase {
                 continue;
             }
             const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString);
-            if (params[i].CacheLocal && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
+            if ((params[i].CacheLocal || this.TrustLocalCacheCompletely) && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
                 const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);
-                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults));
+                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults, results[i].TotalRowCount));
             }
             else if (this.shouldAutoCache(params[i], results[i])) {
                 const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);
-                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults));
+                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults, results[i].TotalRowCount));
                 LogStatusEx({ message: `    📦 [Auto-Cache] RunViews "${params[i].EntityName || params[i].ViewName || 'unknown'}" — ${results[i].Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
             }
             // Register OnDataChanged callback if provided
@@ -1677,24 +1788,8 @@ export class ProviderBase {
      * @param contextUser - The user context for permissions
      */
     async TransformSimpleObjectToEntityObject(param, result, contextUser) {
-        // only if needed (e.g. ResultType==='entity_object'), transform the result set into BaseEntity-derived objects
         if (param.ResultType === 'entity_object' && result && result.Success && result.Results?.length > 0) {
-            // we need to transform each of the items in the result set into a BaseEntity-derived object
-            // Create entities and load data in parallel for better performance
-            const entityPromises = result.Results.map(async (item) => {
-                if (item instanceof BaseEntity || (typeof item.Save === 'function')) {
-                    // the second check is a "duck-typing" check in case we have different runtime
-                    // loading sources where the instanceof will fail
-                    return item;
-                }
-                else {
-                    // not a base entity sub-class already so convert
-                    const entity = await this.GetEntityObject(param.EntityName, contextUser);
-                    await entity.LoadFromData(item);
-                    return entity;
-                }
-            });
-            result.Results = await Promise.all(entityPromises);
+            result.Results = await TransformSimpleObjectToEntityObject(this, param.EntityName, result.Results, contextUser);
         }
     }
     /**