@memberjunction/generic-database-provider 5.37.0 → 5.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/GenericDatabaseProvider.d.ts +46 -26
- package/dist/GenericDatabaseProvider.d.ts.map +1 -1
- package/dist/GenericDatabaseProvider.js +187 -238
- package/dist/GenericDatabaseProvider.js.map +1 -1
- package/dist/queryCompositionEngine.d.ts +23 -4
- package/dist/queryCompositionEngine.d.ts.map +1 -1
- package/dist/queryCompositionEngine.js +27 -18
- package/dist/queryCompositionEngine.js.map +1 -1
- package/dist/queryPagingEngine.d.ts +22 -18
- package/dist/queryPagingEngine.d.ts.map +1 -1
- package/dist/queryPagingEngine.js +71 -195
- package/dist/queryPagingEngine.js.map +1 -1
- package/dist/renderPipeline.d.ts +32 -2
- package/dist/renderPipeline.d.ts.map +1 -1
- package/dist/renderPipeline.js +47 -0
- package/dist/renderPipeline.js.map +1 -1
- package/package.json +14 -13
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*
|
|
16
16
|
* @module @memberjunction/generic-database-provider
|
|
17
17
|
*/
|
|
18
|
-
import { DatabaseProviderBase, EntityFieldTSType, EntityPermissionType, InMemoryLocalStorageProvider, Metadata, RunView,
|
|
18
|
+
import { DatabaseProviderBase, EntityFieldTSType, EntityPermissionType, InMemoryLocalStorageProvider, Metadata, RunView, LocalCacheManager, LogError, LogStatus, LogStatusEx, StripStopWords, AfterKeyNotSupportedError, IsKeysetPaginationOrderableType, } from '@memberjunction/core';
|
|
19
19
|
import { MJGlobal, SQLExpressionValidator, UUIDsEqual } from '@memberjunction/global';
|
|
20
20
|
import { QueryPagingEngine } from './queryPagingEngine.js';
|
|
21
21
|
// QueryParameterProcessor is now called internally by RenderPipeline
|
|
@@ -26,6 +26,7 @@ import { RenderPipeline } from './renderPipeline.js';
|
|
|
26
26
|
import { useJsonArgShape } from './crudSprocFieldRules.js';
|
|
27
27
|
import { QueryEngine, ViewInfo, } from '@memberjunction/core-entities';
|
|
28
28
|
import { AIEngine } from '@memberjunction/aiengine';
|
|
29
|
+
import { SimpleVectorServiceProvider } from '@memberjunction/ai-vectors-memory';
|
|
29
30
|
import { QueueManager } from '@memberjunction/queue';
|
|
30
31
|
import { EntityActionEngineServer } from '@memberjunction/actions';
|
|
31
32
|
import { EncryptionEngine } from '@memberjunction/encryption';
|
|
@@ -43,14 +44,6 @@ const GEO_EXTENDED_TYPES = new Set([
|
|
|
43
44
|
'GeoCountry', 'GeoPostalCode', 'GeoLatitude', 'GeoLongitude'
|
|
44
45
|
]);
|
|
45
46
|
export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
46
|
-
constructor() {
|
|
47
|
-
// Composition engine is now owned by RenderPipeline
|
|
48
|
-
super(...arguments);
|
|
49
|
-
/**************************************************************************/
|
|
50
|
-
// InternalRunQuery — Shared Pipeline Implementation
|
|
51
|
-
/**************************************************************************/
|
|
52
|
-
this._queryCacheInitialized = false;
|
|
53
|
-
}
|
|
54
47
|
/**
|
|
55
48
|
* Returns the active local storage provider, lazily creating an
|
|
56
49
|
* {@link InMemoryLocalStorageProvider} if none has been set.
|
|
@@ -675,6 +668,24 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
675
668
|
getDialect() {
|
|
676
669
|
return null;
|
|
677
670
|
}
|
|
671
|
+
/**
|
|
672
|
+
* Detects infrastructure-level connection errors (timeout, refused, pool closed)
|
|
673
|
+
* as opposed to query-level errors (bad SQL, constraint violations).
|
|
674
|
+
* Delegates to the dialect's driver-specific error classification, with a
|
|
675
|
+
* fallback for POOL_CLOSED errors thrown by our own code.
|
|
676
|
+
*/
|
|
677
|
+
isConnectionError(e) {
|
|
678
|
+
// Dialect-specific check (mssql ConnectionError, pg network codes, etc.)
|
|
679
|
+
if (this.getDialect()?.IsConnectionError(e))
|
|
680
|
+
return true;
|
|
681
|
+
// Our own pool-closed errors are not dialect-specific
|
|
682
|
+
if (e instanceof Error) {
|
|
683
|
+
const code = e.code ?? '';
|
|
684
|
+
if (code === 'POOL_CLOSED')
|
|
685
|
+
return true;
|
|
686
|
+
}
|
|
687
|
+
return false;
|
|
688
|
+
}
|
|
678
689
|
/**
|
|
679
690
|
* Returns the batch separator token for the underlying database platform by delegating to
|
|
680
691
|
* the SQLDialect instance returned by `getDialect()`.
|
|
@@ -1128,13 +1139,11 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
1128
1139
|
whereSQL = bHasWhere ? `${whereSQL} AND (${sExcludeSQL})` : `(${sExcludeSQL})`;
|
|
1129
1140
|
bHasWhere = true;
|
|
1130
1141
|
}
|
|
1131
|
-
// 5. Row-Level Security
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
bHasWhere = true;
|
|
1137
|
-
}
|
|
1142
|
+
// 5. Row-Level Security (exemption check is centralized in GetUserRowLevelSecurityWhereClause)
|
|
1143
|
+
const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Read, '');
|
|
1144
|
+
if (rlsWhereClause && rlsWhereClause.length > 0) {
|
|
1145
|
+
whereSQL = bHasWhere ? `${whereSQL} AND (${rlsWhereClause})` : `(${rlsWhereClause})`;
|
|
1146
|
+
bHasWhere = true;
|
|
1138
1147
|
}
|
|
1139
1148
|
// 6. Keyset (AfterKey) seek predicate — only on the data query, NOT the count query.
|
|
1140
1149
|
// The count is the total matching the user-visible filters; the seek predicate is
|
|
@@ -1291,6 +1300,12 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
1291
1300
|
};
|
|
1292
1301
|
}
|
|
1293
1302
|
catch (e) {
|
|
1303
|
+
// Re-throw infrastructure errors (connection timeout, pool closed, etc.)
|
|
1304
|
+
// so callers can distinguish "database is unreachable" from "query returned
|
|
1305
|
+
// no results." Only query-level errors are safe to return as { Success: false }.
|
|
1306
|
+
if (this.isConnectionError(e)) {
|
|
1307
|
+
throw e;
|
|
1308
|
+
}
|
|
1294
1309
|
const exceptionStopTime = new Date();
|
|
1295
1310
|
LogError(e);
|
|
1296
1311
|
return {
|
|
@@ -1750,11 +1765,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
1750
1765
|
bHasWhere = true;
|
|
1751
1766
|
}
|
|
1752
1767
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
whereSQL = bHasWhere ? `${whereSQL} AND (${rlsWhereClause})` : `(${rlsWhereClause})`;
|
|
1757
|
-
}
|
|
1768
|
+
const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Read, '');
|
|
1769
|
+
if (rlsWhereClause && rlsWhereClause.length > 0) {
|
|
1770
|
+
whereSQL = bHasWhere ? `${whereSQL} AND (${rlsWhereClause})` : `(${rlsWhereClause})`;
|
|
1758
1771
|
}
|
|
1759
1772
|
return whereSQL;
|
|
1760
1773
|
}
|
|
@@ -1960,45 +1973,45 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
1960
1973
|
const errorResults = [];
|
|
1961
1974
|
for (let i = 0; i < params.length; i++) {
|
|
1962
1975
|
const item = params[i];
|
|
1963
|
-
const
|
|
1964
|
-
if (!
|
|
1976
|
+
const query = this.resolveQuery(item.params);
|
|
1977
|
+
if (!query) {
|
|
1965
1978
|
errorResults.push({ queryIndex: i, queryId: item.params.QueryID || '', status: 'error', errorMessage: `Query not found: ${item.params.QueryID || item.params.QueryName}` });
|
|
1966
1979
|
continue;
|
|
1967
1980
|
}
|
|
1968
|
-
if (!
|
|
1969
|
-
errorResults.push({ queryIndex: i, queryId:
|
|
1981
|
+
if (!query.UserCanRun(user)) {
|
|
1982
|
+
errorResults.push({ queryIndex: i, queryId: query.ID, status: 'error', errorMessage: `User does not have permission to run query: ${query.Name}` });
|
|
1970
1983
|
continue;
|
|
1971
1984
|
}
|
|
1972
1985
|
if (!item.cacheStatus) {
|
|
1973
1986
|
itemsWithoutCacheCheck.push({ index: i, item });
|
|
1974
1987
|
continue;
|
|
1975
1988
|
}
|
|
1976
|
-
if (!
|
|
1977
|
-
itemsWithoutValidationSQL.push({ index: i, item,
|
|
1989
|
+
if (!query.CacheValidationSQL) {
|
|
1990
|
+
itemsWithoutValidationSQL.push({ index: i, item, query });
|
|
1978
1991
|
continue;
|
|
1979
1992
|
}
|
|
1980
|
-
itemsNeedingCacheCheck.push({ index: i, item,
|
|
1993
|
+
itemsNeedingCacheCheck.push({ index: i, item, query });
|
|
1981
1994
|
}
|
|
1982
1995
|
const cacheStatusResults = await this.getBatchedQueryCacheStatus(itemsNeedingCacheCheck, contextUser);
|
|
1983
1996
|
const staleItems = [];
|
|
1984
1997
|
const currentResults = [];
|
|
1985
|
-
for (const { index, item,
|
|
1998
|
+
for (const { index, item, query } of itemsNeedingCacheCheck) {
|
|
1986
1999
|
const serverStatus = cacheStatusResults.get(index);
|
|
1987
2000
|
if (!serverStatus || !serverStatus.success) {
|
|
1988
|
-
errorResults.push({ queryIndex: index, queryId:
|
|
2001
|
+
errorResults.push({ queryIndex: index, queryId: query.ID, status: 'error', errorMessage: serverStatus?.errorMessage || 'Failed to get cache status' });
|
|
1989
2002
|
continue;
|
|
1990
2003
|
}
|
|
1991
2004
|
if (this.isCacheCurrent(item.cacheStatus, serverStatus)) {
|
|
1992
|
-
currentResults.push({ queryIndex: index, queryId:
|
|
2005
|
+
currentResults.push({ queryIndex: index, queryId: query.ID, status: 'current' });
|
|
1993
2006
|
}
|
|
1994
2007
|
else {
|
|
1995
|
-
staleItems.push({ index, params: item.params,
|
|
2008
|
+
staleItems.push({ index, params: item.params, query });
|
|
1996
2009
|
}
|
|
1997
2010
|
}
|
|
1998
2011
|
const fullQueryPromises = [
|
|
1999
2012
|
...itemsWithoutCacheCheck.map(({ index, item }) => this.runFullQueryAndReturnForQuery(item.params, index, 'stale', contextUser)),
|
|
2000
|
-
...itemsWithoutValidationSQL.map(({ index, item,
|
|
2001
|
-
...staleItems.map(({ index, params: queryParams,
|
|
2013
|
+
...itemsWithoutValidationSQL.map(({ index, item, query }) => this.runFullQueryAndReturnForQuery(item.params, index, 'no_validation', contextUser, query.ID)),
|
|
2014
|
+
...staleItems.map(({ index, params: queryParams, query }) => this.runFullQueryAndReturnForQuery(queryParams, index, 'stale', contextUser, query.ID)),
|
|
2002
2015
|
];
|
|
2003
2016
|
const fullQueryResults = await Promise.all(fullQueryPromises);
|
|
2004
2017
|
const allResults = [...errorResults, ...currentResults, ...fullQueryResults];
|
|
@@ -2011,69 +2024,42 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2011
2024
|
}
|
|
2012
2025
|
}
|
|
2013
2026
|
/**
|
|
2014
|
-
* Resolves
|
|
2015
|
-
*
|
|
2016
|
-
*/
|
|
2017
|
-
resolveQueryInfo(params) {
|
|
2018
|
-
const freshEntity = this.findQueryInEngine(params.QueryID, params.QueryName, params.CategoryID, params.CategoryPath);
|
|
2019
|
-
if (freshEntity)
|
|
2020
|
-
return this.refreshQueryInfoFromEntity(freshEntity);
|
|
2021
|
-
if (params.QueryID)
|
|
2022
|
-
return this.Queries.find(q => UUIDsEqual(q.ID, params.QueryID));
|
|
2023
|
-
if (params.QueryName) {
|
|
2024
|
-
const matchingQueries = this.Queries.filter(q => q.Name.trim().toLowerCase() === params.QueryName?.trim().toLowerCase());
|
|
2025
|
-
if (matchingQueries.length === 0)
|
|
2026
|
-
return undefined;
|
|
2027
|
-
if (matchingQueries.length === 1)
|
|
2028
|
-
return matchingQueries[0];
|
|
2029
|
-
if (params.CategoryPath) {
|
|
2030
|
-
const byPath = matchingQueries.find(q => q.CategoryPath.toLowerCase() === params.CategoryPath?.toLowerCase());
|
|
2031
|
-
if (byPath)
|
|
2032
|
-
return byPath;
|
|
2033
|
-
}
|
|
2034
|
-
if (params.CategoryID) {
|
|
2035
|
-
const byId = matchingQueries.find(q => UUIDsEqual(q.CategoryID, params.CategoryID));
|
|
2036
|
-
if (byId)
|
|
2037
|
-
return byId;
|
|
2038
|
-
}
|
|
2039
|
-
return matchingQueries[0];
|
|
2040
|
-
}
|
|
2041
|
-
return undefined;
|
|
2042
|
-
}
|
|
2043
|
-
/**
|
|
2044
|
-
* Searches QueryEngine for a fresh query entity.
|
|
2027
|
+
* Resolves a query from RunQueryParams (by ID or Name+CategoryPath).
|
|
2028
|
+
* Uses QueryEngine as the single source of truth for query metadata.
|
|
2045
2029
|
*/
|
|
2046
|
-
|
|
2030
|
+
resolveQuery(params) {
|
|
2047
2031
|
const engineQueries = QueryEngine.Instance?.Queries;
|
|
2048
2032
|
if (!engineQueries || engineQueries.length === 0)
|
|
2049
|
-
return
|
|
2050
|
-
if (QueryID) {
|
|
2051
|
-
|
|
2052
|
-
return engineQueries.find(q => q.ID.trim().toLowerCase() === lower) ?? null;
|
|
2033
|
+
return undefined;
|
|
2034
|
+
if (params.QueryID) {
|
|
2035
|
+
return engineQueries.find(q => UUIDsEqual(q.ID, params.QueryID));
|
|
2053
2036
|
}
|
|
2054
|
-
if (QueryName) {
|
|
2055
|
-
const lowerName = QueryName.trim().toLowerCase();
|
|
2037
|
+
if (params.QueryName) {
|
|
2038
|
+
const lowerName = params.QueryName.trim().toLowerCase();
|
|
2056
2039
|
const matches = engineQueries.filter(q => q.Name.trim().toLowerCase() === lowerName);
|
|
2057
2040
|
if (matches.length === 0)
|
|
2058
|
-
return
|
|
2041
|
+
return undefined;
|
|
2059
2042
|
if (matches.length === 1)
|
|
2060
2043
|
return matches[0];
|
|
2061
|
-
if (
|
|
2062
|
-
const
|
|
2044
|
+
if (params.CategoryPath) {
|
|
2045
|
+
const byPath = matches.find(q => q.CategoryPath.toLowerCase() === params.CategoryPath?.toLowerCase());
|
|
2046
|
+
if (byPath)
|
|
2047
|
+
return byPath;
|
|
2048
|
+
}
|
|
2049
|
+
if (params.CategoryID) {
|
|
2050
|
+
const byId = matches.find(q => UUIDsEqual(q.CategoryID, params.CategoryID));
|
|
2063
2051
|
if (byId)
|
|
2064
2052
|
return byId;
|
|
2065
|
-
|
|
2066
|
-
if (CategoryPath) {
|
|
2067
|
-
const resolvedCategoryId = this.resolveCategoryPath(CategoryPath);
|
|
2053
|
+
const resolvedCategoryId = this.resolveCategoryPath(params.CategoryPath ?? '');
|
|
2068
2054
|
if (resolvedCategoryId) {
|
|
2069
|
-
const
|
|
2070
|
-
if (
|
|
2071
|
-
return
|
|
2055
|
+
const byResolvedPath = matches.find(q => UUIDsEqual(q.CategoryID, resolvedCategoryId));
|
|
2056
|
+
if (byResolvedPath)
|
|
2057
|
+
return byResolvedPath;
|
|
2072
2058
|
}
|
|
2073
2059
|
}
|
|
2074
2060
|
return matches[0];
|
|
2075
2061
|
}
|
|
2076
|
-
return
|
|
2062
|
+
return undefined;
|
|
2077
2063
|
}
|
|
2078
2064
|
/**
|
|
2079
2065
|
* Validates that a query can be executed by the given user. Checks both permissions
|
|
@@ -2081,7 +2067,7 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2081
2067
|
* emits a console warning but allows execution to proceed, enabling query testing
|
|
2082
2068
|
* before formal approval.
|
|
2083
2069
|
*
|
|
2084
|
-
* @param query - The resolved
|
|
2070
|
+
* @param query - The resolved MJQueryEntityExtended to validate
|
|
2085
2071
|
* @param contextUser - The user attempting to execute the query
|
|
2086
2072
|
* @throws Error if the user does not have permission to run the query
|
|
2087
2073
|
*/
|
|
@@ -2094,20 +2080,6 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2094
2080
|
LogStatus(`WARNING: Executing query '${query.Name}' (ID: ${query.ID}) with status '${query.Status}'. Query has not been approved.`);
|
|
2095
2081
|
}
|
|
2096
2082
|
}
|
|
2097
|
-
/**
|
|
2098
|
-
* Creates a fresh QueryInfo from a MJQueryEntity and patches the ProviderBase cache.
|
|
2099
|
-
*/
|
|
2100
|
-
refreshQueryInfoFromEntity(entity) {
|
|
2101
|
-
const freshInfo = new QueryInfo(entity.GetAll());
|
|
2102
|
-
const existingIndex = this.Queries.findIndex(q => UUIDsEqual(q.ID, freshInfo.ID));
|
|
2103
|
-
if (existingIndex >= 0) {
|
|
2104
|
-
this.Queries[existingIndex] = freshInfo;
|
|
2105
|
-
}
|
|
2106
|
-
else {
|
|
2107
|
-
this.Queries.push(freshInfo);
|
|
2108
|
-
}
|
|
2109
|
-
return freshInfo;
|
|
2110
|
-
}
|
|
2111
2083
|
/**
|
|
2112
2084
|
* Resolves a category path string to a QueryCategoryInfo ID.
|
|
2113
2085
|
*/
|
|
@@ -2134,9 +2106,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2134
2106
|
const results = new Map();
|
|
2135
2107
|
if (items.length === 0)
|
|
2136
2108
|
return results;
|
|
2137
|
-
const promises = items.map(async ({ index,
|
|
2109
|
+
const promises = items.map(async ({ index, query }) => {
|
|
2138
2110
|
try {
|
|
2139
|
-
const rows = await this.ExecuteSQL(
|
|
2111
|
+
const rows = await this.ExecuteSQL(query.CacheValidationSQL, undefined, undefined, contextUser);
|
|
2140
2112
|
if (rows && rows.length > 0) {
|
|
2141
2113
|
const row = rows[0];
|
|
2142
2114
|
results.set(index, {
|
|
@@ -2179,13 +2151,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2179
2151
|
rowCount: result.Results.length,
|
|
2180
2152
|
};
|
|
2181
2153
|
}
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
this._queryCacheInitialized = true;
|
|
2186
|
-
}
|
|
2187
|
-
return QueryCacheManager.Instance;
|
|
2188
|
-
}
|
|
2154
|
+
/**************************************************************************/
|
|
2155
|
+
// InternalRunQuery — Shared Pipeline Implementation
|
|
2156
|
+
/**************************************************************************/
|
|
2189
2157
|
/**
|
|
2190
2158
|
* Full query execution pipeline: resolve → validate → compose → template → cache check →
|
|
2191
2159
|
* execute → paginate → audit → cache store. Platform providers inherit this; only
|
|
@@ -2203,7 +2171,6 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2203
2171
|
const { finalSQL, appliedParameters } = this.processQueryParameters(query, params.Parameters, contextUser);
|
|
2204
2172
|
// Execute query — use SQL-level paging when requested, else fetch all rows
|
|
2205
2173
|
const useSQLPaging = QueryPagingEngine.ShouldPage(params.StartRow, params.MaxRows);
|
|
2206
|
-
const cacheConfig = query.CacheConfig;
|
|
2207
2174
|
let paginatedResult;
|
|
2208
2175
|
let totalRowCount;
|
|
2209
2176
|
let executionTime;
|
|
@@ -2215,43 +2182,19 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2215
2182
|
return pagedCacheHit;
|
|
2216
2183
|
}
|
|
2217
2184
|
const paging = QueryPagingEngine.WrapWithPaging(finalSQL, params.StartRow, params.MaxRows, this.PlatformKey);
|
|
2218
|
-
//
|
|
2219
|
-
const cachedCount = cacheConfig?.enabled
|
|
2220
|
-
? await this.QueryCacheMgr.GetTotalRowCount(query, params.Parameters || {})
|
|
2221
|
-
: null;
|
|
2185
|
+
// Execute data + count queries in parallel
|
|
2222
2186
|
const start = Date.now();
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
const [dataResult, countResult] = await Promise.all([
|
|
2235
|
-
this.ExecuteSQL(paging.DataSQL, undefined, undefined, contextUser),
|
|
2236
|
-
this.ExecuteSQL(paging.CountSQL, undefined, undefined, contextUser),
|
|
2237
|
-
]);
|
|
2238
|
-
executionTime = Date.now() - start;
|
|
2239
|
-
if (!dataResult)
|
|
2240
|
-
throw new Error('Error executing paged query SQL');
|
|
2241
|
-
paginatedResult = dataResult;
|
|
2242
|
-
totalRowCount = countResult?.[0]?.TotalRowCount != null
|
|
2243
|
-
? Number(countResult[0].TotalRowCount)
|
|
2244
|
-
: paginatedResult.length;
|
|
2245
|
-
// Cache the count for subsequent page requests (fire-and-forget)
|
|
2246
|
-
if (cacheConfig?.enabled) {
|
|
2247
|
-
void this.QueryCacheMgr.SetTotalRowCount(query, params.Parameters || {}, totalRowCount);
|
|
2248
|
-
}
|
|
2249
|
-
}
|
|
2250
|
-
// Cache the paged results (fire-and-forget)
|
|
2251
|
-
if (cacheConfig?.enabled) {
|
|
2252
|
-
void this.QueryCacheMgr.SetPaged(query, params.Parameters || {}, params.StartRow, params.MaxRows, paginatedResult);
|
|
2253
|
-
void this.QueryCacheMgr.InvalidateWithDependents(query);
|
|
2254
|
-
}
|
|
2187
|
+
const [dataResult, countResult] = await Promise.all([
|
|
2188
|
+
this.ExecuteSQL(paging.DataSQL, undefined, undefined, contextUser),
|
|
2189
|
+
this.ExecuteSQL(paging.CountSQL, undefined, undefined, contextUser),
|
|
2190
|
+
]);
|
|
2191
|
+
executionTime = Date.now() - start;
|
|
2192
|
+
if (!dataResult)
|
|
2193
|
+
throw new Error('Error executing paged query SQL');
|
|
2194
|
+
paginatedResult = dataResult;
|
|
2195
|
+
totalRowCount = countResult?.[0]?.TotalRowCount != null
|
|
2196
|
+
? Number(countResult[0].TotalRowCount)
|
|
2197
|
+
: paginatedResult.length;
|
|
2255
2198
|
}
|
|
2256
2199
|
else {
|
|
2257
2200
|
// Check full-result cache before executing
|
|
@@ -2287,6 +2230,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2287
2230
|
};
|
|
2288
2231
|
}
|
|
2289
2232
|
catch (e) {
|
|
2233
|
+
if (this.isConnectionError(e)) {
|
|
2234
|
+
throw e;
|
|
2235
|
+
}
|
|
2290
2236
|
LogError(e);
|
|
2291
2237
|
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
2292
2238
|
return {
|
|
@@ -2327,30 +2273,7 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2327
2273
|
ErrorMessage: validation.error || 'SQL validation failed',
|
|
2328
2274
|
};
|
|
2329
2275
|
}
|
|
2330
|
-
// Check ad-hoc cache if opt-in TTL is provided
|
|
2331
|
-
const adhocTTL = params.AdhocCacheTTLMinutes;
|
|
2332
|
-
if (adhocTTL != null && adhocTTL > 0) {
|
|
2333
|
-
const cached = await this.QueryCacheMgr.GetAdhoc(params.SQL, adhocTTL);
|
|
2334
|
-
if (cached) {
|
|
2335
|
-
const { paginatedResult, totalRowCount } = this.applyQueryPagination(cached.results, params);
|
|
2336
|
-
return {
|
|
2337
|
-
Success: true,
|
|
2338
|
-
QueryID: '',
|
|
2339
|
-
QueryName: 'Ad-Hoc Query',
|
|
2340
|
-
Results: paginatedResult,
|
|
2341
|
-
RowCount: paginatedResult.length,
|
|
2342
|
-
TotalRowCount: totalRowCount,
|
|
2343
|
-
ExecutionTime: 0,
|
|
2344
|
-
ErrorMessage: '',
|
|
2345
|
-
CacheHit: true,
|
|
2346
|
-
};
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
2276
|
const { result, executionTime } = await this.executeQueryWithTiming(params.SQL, contextUser);
|
|
2350
|
-
// Store in ad-hoc cache if opt-in (fire-and-forget)
|
|
2351
|
-
if (adhocTTL != null && adhocTTL > 0) {
|
|
2352
|
-
void this.QueryCacheMgr.SetAdhoc(params.SQL, adhocTTL, result);
|
|
2353
|
-
}
|
|
2354
2277
|
const { paginatedResult, totalRowCount } = this.applyQueryPagination(result, params);
|
|
2355
2278
|
return {
|
|
2356
2279
|
Success: true,
|
|
@@ -2380,10 +2303,10 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2380
2303
|
}
|
|
2381
2304
|
/**
|
|
2382
2305
|
* Finds a query from RunQueryParams and validates user permissions.
|
|
2383
|
-
* Uses `
|
|
2306
|
+
* Uses `resolveQuery()` for lookup and `ValidateQueryForExecution()` for permissions.
|
|
2384
2307
|
*/
|
|
2385
2308
|
findAndValidateQuery(params, contextUser) {
|
|
2386
|
-
const query = this.
|
|
2309
|
+
const query = this.resolveQuery(params);
|
|
2387
2310
|
if (!query) {
|
|
2388
2311
|
let errorDetails = 'Query not found';
|
|
2389
2312
|
if (params.QueryName) {
|
|
@@ -2415,8 +2338,12 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2415
2338
|
Platform: this.PlatformKey,
|
|
2416
2339
|
ContextUser: contextUser,
|
|
2417
2340
|
Parameters: parameters,
|
|
2418
|
-
UsesTemplate: query.UsesTemplate,
|
|
2419
|
-
QueryInfo:
|
|
2341
|
+
UsesTemplate: query.UsesTemplate ?? false,
|
|
2342
|
+
QueryInfo: {
|
|
2343
|
+
SQL: query.SQL ?? '',
|
|
2344
|
+
UsesTemplate: query.UsesTemplate ?? false,
|
|
2345
|
+
Parameters: query.QueryParameters,
|
|
2346
|
+
},
|
|
2420
2347
|
});
|
|
2421
2348
|
if (!result.HasCompositions && !query.UsesTemplate && parameters && Object.keys(parameters).length > 0) {
|
|
2422
2349
|
LogStatus('Warning: Parameters provided but query does not use templates. Parameters will be ignored.');
|
|
@@ -2474,11 +2401,15 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2474
2401
|
* pipeline. Supports inline dependencies for transient query testing.
|
|
2475
2402
|
*/
|
|
2476
2403
|
resolveSpecParameters(spec, contextUser) {
|
|
2404
|
+
// QueryExecutionSpec.ParameterDefinitions is QueryParameterInfo[] (MJCore),
|
|
2405
|
+
// while RenderContext expects MJQueryParameterEntity[] (core-entities).
|
|
2406
|
+
// Both share the structural shape the processor needs (Name, DataType, IsRequired).
|
|
2407
|
+
const paramDefs = spec.ParameterDefinitions;
|
|
2477
2408
|
const result = RenderPipeline.Run(spec.SQL, {
|
|
2478
2409
|
Platform: this.PlatformKey,
|
|
2479
2410
|
ContextUser: contextUser,
|
|
2480
2411
|
Parameters: spec.Parameters,
|
|
2481
|
-
ParameterDefinitions:
|
|
2412
|
+
ParameterDefinitions: paramDefs,
|
|
2482
2413
|
UsesTemplate: spec.UsesTemplate,
|
|
2483
2414
|
Dependencies: spec.Dependencies,
|
|
2484
2415
|
OriginalSQL: spec.SQL,
|
|
@@ -2492,62 +2423,18 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2492
2423
|
// wrapWithMaxRows is now handled by RenderPipeline.applyMaxRows
|
|
2493
2424
|
/**
|
|
2494
2425
|
* Checks the query cache for existing results and returns them if valid.
|
|
2426
|
+
* Currently always returns null (query caching is not active).
|
|
2495
2427
|
*/
|
|
2496
|
-
async checkQueryCache(
|
|
2497
|
-
|
|
2498
|
-
if (!cacheConfig?.enabled) {
|
|
2499
|
-
return null;
|
|
2500
|
-
}
|
|
2501
|
-
const cached = await this.QueryCacheMgr.Get(query, params.Parameters || {});
|
|
2502
|
-
if (!cached) {
|
|
2503
|
-
return null;
|
|
2504
|
-
}
|
|
2505
|
-
LogStatus(`Cache hit for query ${query.Name} (${query.ID})`);
|
|
2506
|
-
const { paginatedResult, totalRowCount } = this.applyQueryPagination(cached.results, params);
|
|
2507
|
-
return {
|
|
2508
|
-
Success: true,
|
|
2509
|
-
QueryID: query.ID,
|
|
2510
|
-
QueryName: query.Name,
|
|
2511
|
-
Results: paginatedResult,
|
|
2512
|
-
RowCount: paginatedResult.length,
|
|
2513
|
-
TotalRowCount: totalRowCount,
|
|
2514
|
-
ExecutionTime: 0,
|
|
2515
|
-
ErrorMessage: '',
|
|
2516
|
-
AppliedParameters: appliedParameters,
|
|
2517
|
-
CacheHit: true,
|
|
2518
|
-
CacheTTLRemaining: cached.ttlRemainingMs,
|
|
2519
|
-
};
|
|
2428
|
+
async checkQueryCache(_query, _params, _appliedParameters) {
|
|
2429
|
+
return null;
|
|
2520
2430
|
}
|
|
2521
2431
|
/**
|
|
2522
2432
|
* Checks the paged cache for a specific page of query results.
|
|
2523
2433
|
* Returns a full RunQueryResult on hit, null on miss.
|
|
2434
|
+
* Currently always returns null (query caching is not active).
|
|
2524
2435
|
*/
|
|
2525
|
-
async checkPagedQueryCache(
|
|
2526
|
-
|
|
2527
|
-
if (!cacheConfig?.enabled)
|
|
2528
|
-
return null;
|
|
2529
|
-
const cached = await this.QueryCacheMgr.GetPaged(query, params.Parameters || {}, params.StartRow, params.MaxRows);
|
|
2530
|
-
if (!cached)
|
|
2531
|
-
return null;
|
|
2532
|
-
// Also try to get the cached count
|
|
2533
|
-
const cachedCount = await this.QueryCacheMgr.GetTotalRowCount(query, params.Parameters || {});
|
|
2534
|
-
const totalRowCount = cachedCount ?? cached.results.length;
|
|
2535
|
-
LogStatus(`Paged cache hit for query ${query.Name} (${query.ID}) page ${params.StartRow}+${params.MaxRows}`);
|
|
2536
|
-
return {
|
|
2537
|
-
Success: true,
|
|
2538
|
-
QueryID: query.ID,
|
|
2539
|
-
QueryName: query.Name,
|
|
2540
|
-
Results: cached.results,
|
|
2541
|
-
RowCount: cached.results.length,
|
|
2542
|
-
TotalRowCount: totalRowCount,
|
|
2543
|
-
PageNumber: Math.floor(params.StartRow / params.MaxRows) + 1,
|
|
2544
|
-
PageSize: params.MaxRows,
|
|
2545
|
-
ExecutionTime: 0,
|
|
2546
|
-
ErrorMessage: '',
|
|
2547
|
-
AppliedParameters: appliedParameters,
|
|
2548
|
-
CacheHit: true,
|
|
2549
|
-
CacheTTLRemaining: cached.ttlRemainingMs,
|
|
2550
|
-
};
|
|
2436
|
+
async checkPagedQueryCache(_query, _params, _appliedParameters) {
|
|
2437
|
+
return null;
|
|
2551
2438
|
}
|
|
2552
2439
|
/**
|
|
2553
2440
|
* Executes the query SQL and tracks execution time.
|
|
@@ -2603,16 +2490,10 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2603
2490
|
}
|
|
2604
2491
|
/**
|
|
2605
2492
|
* Caches query results if caching is enabled for the query.
|
|
2606
|
-
*
|
|
2493
|
+
* Currently a no-op (query caching is not active).
|
|
2607
2494
|
*/
|
|
2608
|
-
async cacheQueryResults(
|
|
2609
|
-
|
|
2610
|
-
if (!cacheConfig?.enabled) {
|
|
2611
|
-
return;
|
|
2612
|
-
}
|
|
2613
|
-
await this.QueryCacheMgr.Set(query, parameters, results);
|
|
2614
|
-
await this.QueryCacheMgr.InvalidateWithDependents(query);
|
|
2615
|
-
LogStatus(`Cached results for query ${query.Name} (${query.ID})`);
|
|
2495
|
+
async cacheQueryResults(_query, _parameters, _results) {
|
|
2496
|
+
// No-op: query caching has been removed
|
|
2616
2497
|
}
|
|
2617
2498
|
/**************************************************************************/
|
|
2618
2499
|
// Load — Shared Implementation
|
|
@@ -2631,9 +2512,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2631
2512
|
const quotes = pk.NeedsQuotes ? "'" : '';
|
|
2632
2513
|
return `${this.QuoteIdentifier(pk.CodeName)}=${quotes}${val.Value}${quotes}`;
|
|
2633
2514
|
}).join(' AND ');
|
|
2634
|
-
// Append Read RLS filter
|
|
2515
|
+
// Append Read RLS filter (exemption check is centralized in GetUserRowLevelSecurityWhereClause)
|
|
2635
2516
|
let fullWhere = where;
|
|
2636
|
-
if (user
|
|
2517
|
+
if (user) {
|
|
2637
2518
|
const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Read, '');
|
|
2638
2519
|
if (rlsWhereClause && rlsWhereClause.length > 0) {
|
|
2639
2520
|
fullWhere = `${where} AND (${rlsWhereClause})`;
|
|
@@ -2693,9 +2574,6 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2693
2574
|
*/
|
|
2694
2575
|
async CheckRecordRLS(entity, user, type) {
|
|
2695
2576
|
const entityInfo = entity.EntityInfo;
|
|
2696
|
-
if (entityInfo.UserExemptFromRowLevelSecurity(user, type)) {
|
|
2697
|
-
return true;
|
|
2698
|
-
}
|
|
2699
2577
|
const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, type, '');
|
|
2700
2578
|
if (!rlsWhereClause || rlsWhereClause.length === 0) {
|
|
2701
2579
|
return true;
|
|
@@ -2715,9 +2593,6 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
2715
2593
|
*/
|
|
2716
2594
|
async CheckCreateRLS(entity, user) {
|
|
2717
2595
|
const entityInfo = entity.EntityInfo;
|
|
2718
|
-
if (entityInfo.UserExemptFromRowLevelSecurity(user, EntityPermissionType.Create)) {
|
|
2719
|
-
return true;
|
|
2720
|
-
}
|
|
2721
2596
|
const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Create, '');
|
|
2722
2597
|
if (!rlsWhereClause || rlsWhereClause.length === 0) {
|
|
2723
2598
|
return true;
|
|
@@ -3198,5 +3073,79 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
|
|
|
3198
3073
|
}
|
|
3199
3074
|
return maxDate ? maxDate.toISOString() : new Date().toISOString();
|
|
3200
3075
|
}
|
|
3076
|
+
/**
|
|
3077
|
+
* Server-side semantic ranking pass for {@link ProviderBase.SearchEntity}
|
|
3078
|
+
* (and, by extension, the batched {@link ProviderBase.SearchEntities}).
|
|
3079
|
+
*
|
|
3080
|
+
* The query embedding MUST be generated with the same model that produced
|
|
3081
|
+
* the indexed vectors — otherwise cosine scores compare apples to oranges
|
|
3082
|
+
* and rankings are garbage. We look up the EntityDocument's `AIModelID` via
|
|
3083
|
+
* `AIEngine.Models` to recover the driver class / APIName and call
|
|
3084
|
+
* `EmbedText(model, text)` directly. If the EntityDocument does not specify
|
|
3085
|
+
* a model (or the model isn't loaded) we fall back to
|
|
3086
|
+
* `EmbedTextLocal` (highest-power local model) only as a last resort.
|
|
3087
|
+
*
|
|
3088
|
+
* Vector ranking runs against the in-process `SimpleVectorServiceProvider`,
|
|
3089
|
+
* which rehydrates the vector pool for `entityDocumentId` from
|
|
3090
|
+
* `MJ: Entity Record Documents.VectorJSON` rows.
|
|
3091
|
+
*
|
|
3092
|
+
* Failures (no embedding model available, vector index miss) degrade to an
|
|
3093
|
+
* empty result set so hybrid mode can still surface lexical matches.
|
|
3094
|
+
*/
|
|
3095
|
+
async searchEntitiesSemanticPass(entityDocumentId, searchText, overFetch, embeddingAIModelId, contextUser) {
|
|
3096
|
+
// Ensure AIEngine is loaded so Models / EmbedText are usable. Config()
|
|
3097
|
+
// is a no-op when already initialized — safe to call on every search.
|
|
3098
|
+
try {
|
|
3099
|
+
await AIEngine.Instance.Config(false, contextUser);
|
|
3100
|
+
}
|
|
3101
|
+
catch (e) {
|
|
3102
|
+
LogError(`searchEntitiesSemanticPass: AIEngine.Config failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3103
|
+
return [];
|
|
3104
|
+
}
|
|
3105
|
+
let queryVector = null;
|
|
3106
|
+
try {
|
|
3107
|
+
if (embeddingAIModelId) {
|
|
3108
|
+
const model = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, embeddingAIModelId));
|
|
3109
|
+
if (!model) {
|
|
3110
|
+
LogError(`searchEntitiesSemanticPass: EntityDocument AIModelID="${embeddingAIModelId}" not found in AIEngine.Models. Index/query model mismatch is likely — refusing to fall back silently.`);
|
|
3111
|
+
return [];
|
|
3112
|
+
}
|
|
3113
|
+
const result = await AIEngine.Instance.EmbedText(model, searchText);
|
|
3114
|
+
queryVector = result?.vector ?? null;
|
|
3115
|
+
}
|
|
3116
|
+
else {
|
|
3117
|
+
// No model on the EntityDocument — last-resort fallback to the
|
|
3118
|
+
// highest-power local embedder. Logged because this path means
|
|
3119
|
+
// indexing-time and query-time models can diverge.
|
|
3120
|
+
LogError(`searchEntitiesSemanticPass: no AIModelID on EntityDocument "${entityDocumentId}"; falling back to highest-power local embedding model. Index/query mismatch possible.`);
|
|
3121
|
+
const embed = await AIEngine.Instance.EmbedTextLocal(searchText);
|
|
3122
|
+
queryVector = embed?.result?.vector ?? null;
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
catch (e) {
|
|
3126
|
+
LogError(`searchEntitiesSemanticPass: embedding generation threw: ${e instanceof Error ? e.message : String(e)}`);
|
|
3127
|
+
return [];
|
|
3128
|
+
}
|
|
3129
|
+
if (!queryVector)
|
|
3130
|
+
return [];
|
|
3131
|
+
const vectorProvider = new SimpleVectorServiceProvider();
|
|
3132
|
+
const result = await vectorProvider.QueryIndex({ id: entityDocumentId, vector: queryVector, topK: overFetch }, contextUser);
|
|
3133
|
+
if (!result.success)
|
|
3134
|
+
return [];
|
|
3135
|
+
const data = result.data;
|
|
3136
|
+
const matches = data?.matches ?? [];
|
|
3137
|
+
const out = [];
|
|
3138
|
+
for (const m of matches) {
|
|
3139
|
+
const recordId = String(m.metadata?.['RecordID'] ?? '');
|
|
3140
|
+
if (!recordId)
|
|
3141
|
+
continue;
|
|
3142
|
+
out.push({
|
|
3143
|
+
ID: recordId,
|
|
3144
|
+
Score: m.score,
|
|
3145
|
+
Metadata: { entityRecordDocumentId: m.id },
|
|
3146
|
+
});
|
|
3147
|
+
}
|
|
3148
|
+
return out;
|
|
3149
|
+
}
|
|
3201
3150
|
}
|
|
3202
3151
|
//# sourceMappingURL=GenericDatabaseProvider.js.map
|