@adaptabletools/adaptable 23.1.0 → 24.0.0-canary.0

package/src/Utilities/Services/QueryLanguageService.js

@@ -1,525 +1,797 @@
-import { isObservable } from 'rxjs';
-import * as parser from '../../parser/src';
-import { aggregatedBooleanExpressionFunctions, } from '../ExpressionFunctions/aggregatedBooleanExpressionFunctions';
-import { addGroupByParams, aggregatedScalarExpressionFunctions, } from '../ExpressionFunctions/aggregatedScalarExpressionFunctions';
-import { booleanExpressionFunctions } from '../ExpressionFunctions/booleanExpressionFunctions';
-import { extractColumnParameters, extractParameter, } from '../ExpressionFunctions/expressionFunctionUtils';
-import { observableExpressionFunctions } from '../ExpressionFunctions/observableExpressionFunctions';
-import { scalarExpressionFunctions } from '../ExpressionFunctions/scalarExpressionFunctions';
+import { acquireObservableSlots, canonicalizeExpression, createAggregationCatalogue, createAqlEngine, DEFAULT_ROW_SET_KIND, ExpressionEvaluationError, findDependencyCycle, getNodesFromExpression, hostGetFirstRowRef, mapDataTypeToStaticType, parse, validateAqlExpression, validateExpressionTypes, } from "../../../vendor/aql/index.js";
+import { createComputedValueStore, createHistoryStore, createSystemClock, } from "../../../vendor/aql/store/index.js";
+import { inferExpressionKindFromText, } from '../../AdaptableState/Common/AdaptableQuery';
+import { applyExpressionFunctionAvailability } from '../../aql/applyExpressionFunctionAvailability';
+import { createAdaptableAqlHost, toAqlCellChange } from '../../aql/createAdaptableAqlHost';
+import { moduleAllowsObservableCalls } from '../../aql/expressionFunctionAvailability';
+import { mapLegacyKindToValidationRequest } from '../../aql/legacyExpressionKinds';
+import { mapCustomAggregatedFunctionsToOperators, toCustomAggregationFunctionDef, } from '../../aql/mapCustomAggregatedScalarFunctions';
+import { aggregationLookupFunctions, systemAggregatedExpressionFunctions, systemAggregationClauseFunctions, systemScalarExpressionFunctions, systemObservableExpressionFunctions, } from '../../aql/systemExpressionFunctions';
+import { PROTECTED_CORE_EXPRESSION_FUNCTIONS } from '../../aql/resolveCustomExpressionFunctions';
+import { resolveCustomExpressionFunctions } from '../../aql/resolveCustomExpressionFunctions';
 import { getTypedKeys } from '../Extensions/TypeExtensions';
-import { AggregatedScalarLiveValue } from './AggregatedScalarLiveValue';
+const VALID = { isValid: true, errorMessage: '' };
+const ROW_EVALUATOR_CACHE_SIZE = 64;
+const CANONICAL_TEXT_CACHE_SIZE = 1024;
+const DEFAULT_MAX_WINDOW_MS = 28_800_000;
+const DEFAULT_MAX_HISTORY_PER_CELL = 1000;
+const toChangeTrigger = (trigger) => {
+    switch (trigger) {
+        case 'calculatedColumnChange':
+            return 'calculated';
+        case 'tick':
+        case 'undo':
+            return trigger;
+        default:
+            return 'edit';
+    }
+};
+const toErrorMessage = (error) => {
+    if (error instanceof ExpressionEvaluationError) {
+        return `${error.expressionFnName} ${error.message}`;
+    }
+    if (error instanceof Error) {
+        return error.message;
+    }
+    return String(error);
+};
+const invalid = (errorMessage) => ({
+    isValid: false,
+    errorMessage: typeof errorMessage === 'string' ? errorMessage : toErrorMessage(errorMessage),
+});
+const getCustomPoolNames = (pool, systemPool) => (pool ? Object.keys(pool).filter((name) => !(name in systemPool)) : []);
+const throwIfCycle = (cycle) => {
+    if (cycle) {
+        throw new ExpressionEvaluationError(`${cycle[0]}`, ` contains a circular reference: ${cycle.join('  ->  ')}`);
+    }
+};
 export class QueryLanguageService {
     adaptableApi;
-    cacheBooleanValidation = new Map();
-    cacheObservableValidation = new Map();
-    cacheAggregatedBooleanValidation = new Map();
-    cacheAggregatedScalarValidation = new Map();
-    cacheModuleSpecificExpressionFunctions = new Map();
+    validationCache = new Map();
+    expressionKindCache = new Map();
+    moduleFunctionsCache = new Map();
+    aggregationCatalogueCache = new Map();
+    engineCache = new Map();
+    hostEngineCache = new WeakMap();
+    rowEvaluatorCache = new Map();
+    hostRowEvaluatorCache = new WeakMap();
+    canonicalTextCache = new Map();
+    functionsGeneration = 0;
+    _aqlHost;
+    _clock;
+    _historyStore;
+    _computedValueStore;
+    dependenciesCache = new Map();
+    observableSeriesReleases = new Map();
+    hostEventsBound = false;
+    hostEventUnsubscribers = [];
     constructor(adaptableApi) {
         this.adaptableApi = adaptableApi;
     }
-    evaluateBooleanExpression(expression, module, rowNode, evalContext) {
-        if (expression == undefined) {
-            this.adaptableApi.logError('QueryLanguageService.evaluateBooleanExpression was called with an undefined expression');
+    destroy() {
+        this.hostEventUnsubscribers.forEach((unsubscribe) => {
+            try {
+                unsubscribe();
+            }
+            catch {
+            }
+        });
+        this.hostEventUnsubscribers = [];
+        this.hostEventsBound = false;
+        this._computedValueStore?.destroy();
+        this._computedValueStore = undefined;
+        this.releaseAllObservableSeries();
+        this._historyStore?.destroy();
+        this._historyStore = undefined;
+        this.dependenciesCache.clear();
+        this.validationCache.clear();
+        this.expressionKindCache.clear();
+        this.rowEvaluatorCache.clear();
+        this.aggregationCatalogueCache.clear();
+        this.engineCache.clear();
+    }
+    toCacheExpression(expression) {
+        const cached = this.canonicalTextCache.get(expression);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const canonical = canonicalizeExpression(expression);
+        if (this.canonicalTextCache.size >= CANONICAL_TEXT_CACHE_SIZE) {
+            const oldestKey = this.canonicalTextCache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.canonicalTextCache.delete(oldestKey);
+            }
+        }
+        this.canonicalTextCache.set(expression, canonical);
+        return canonical;
+    }
+    get aqlHost() {
+        return (this._aqlHost ??= createAdaptableAqlHost(this.adaptableApi));
+    }
+    getClock() {
+        return (this._clock ??= createSystemClock());
+    }
+    getHistoryStore() {
+        if (!this._historyStore) {
+            const expressionOptions = this.adaptableApi.optionsApi.getExpressionOptions();
+            this._historyStore = createHistoryStore({
+                clock: this.getClock(),
+                getRowId: (rowRef) => rowRef?.id,
+                maxWindowMs: expressionOptions.maxTimeframeSize ?? DEFAULT_MAX_WINDOW_MS,
+                maxEntriesPerCell: expressionOptions.maxHistoryPerCell ?? DEFAULT_MAX_HISTORY_PER_CELL,
+                evaluateFilter: (filterKey, rowRef) => this.evaluateHistoryFilter(filterKey, rowRef),
+            });
+            this.bindHostEvents();
+        }
+        return this._historyStore;
+    }
+    evaluateHistoryFilter(filterKey, rowNode) {
+        try {
+            return Boolean(this.getPreparedRowEvaluator(filterKey, 'Expression')(rowNode));
+        }
+        catch {
             return false;
         }
-        const moduleExpressionFunctions = this.getModuleExpressionFunctionsMap(module);
-        const booleanAndScalarFunctions = this.getBooleanAndScalarFunctions(moduleExpressionFunctions);
-        return parser.evaluate(expression, {
-            node: rowNode,
-            functions: booleanAndScalarFunctions,
-            evaluateCustomQueryVariable: this.evaluateCustomQueryVariable,
-            dataChangedEvent: evalContext?.dataChangedEvent,
-            pivotResultColumn: evalContext?.pivotResultColumn,
-            columnScope: evalContext?.columnScope,
-            ...this.adaptableApi.internalApi.buildBaseContext(),
-        });
     }
-    evaluateScalarExpression(expression, module, rowNode) {
-        return this.evaluateBooleanExpression(expression, module, rowNode);
+    toPerRowConfig(config) {
+        const pivotBaseColumnId = config.pivotResultColumn?.getColDef()?.pivotValueColumn?.getColId();
+        return {
+            rowRef: config.rowNode,
+            columnScope: config.columnScope,
+            cellValueOverride: config.cellValueOverride,
+            columnValueAliases: pivotBaseColumnId
+                ? { [pivotBaseColumnId]: config.pivotResultColumn.getColId() }
+                : undefined,
+            cellDataChange: config.dataChangedEvent
+                ? toAqlCellChange(config.dataChangedEvent)
+                : undefined,
+        };
     }
-    evaluateAggregatedScalarExpression(expression, module, getRowNodes) {
-        const aggregatedScalarFunctions = this.getModuleExpressionFunctionsMap(module).aggregatedScalarFunctions;
-        return parser.evaluate(expression, {
-            node: null,
-            functions: aggregatedScalarFunctions,
-            evaluateCustomQueryVariable: this.evaluateCustomQueryVariable,
-            getRowNodes,
-            ...this.adaptableApi.internalApi.buildBaseContext(),
-        });
+    toEvaluationConfig(config) {
+        return {
+            ...this.toPerRowConfig(config),
+            getRowRefs: config.getRowNodes,
+            ext: this.adaptableApi.internalApi.buildBaseContext(),
+        };
     }
-    evaluateObservableExpression(reactiveExpression, module) {
-        const moduleExpressionFunctions = this.getModuleExpressionFunctionsMap(module);
-        const booleanAndScalarFunctions = this.getBooleanAndScalarFunctions(moduleExpressionFunctions);
-        const reactiveExpression$ = parser.evaluate(reactiveExpression, {
-            node: this.adaptableApi.gridApi.getFirstRowNode(),
-            functions: moduleExpressionFunctions.observableFunctions,
-            whereClauseFunctions: booleanAndScalarFunctions,
-            evaluateCustomQueryVariable: this.evaluateCustomQueryVariable,
-            ...this.adaptableApi.internalApi.buildBaseContext(),
-        });
-        return reactiveExpression$;
-    }
-    evaluateAggregatedBooleanExpression(aggregationExpression, module) {
-        const moduleExpressionFunctionsMap = this.getModuleExpressionFunctionsMap(module);
-        const booleanAndScalarFunctions = this.getBooleanAndScalarFunctions(moduleExpressionFunctionsMap);
-        const aggregationEvaluation = parser.evaluate(aggregationExpression, {
-            node: this.adaptableApi.gridApi.getFirstRowNode(),
-            functions: moduleExpressionFunctionsMap.aggregatedBooleanFunctions,
-            whereClauseFunctions: booleanAndScalarFunctions,
-            evaluateCustomQueryVariable: this.evaluateCustomQueryVariable,
-            ...this.adaptableApi.internalApi.buildBaseContext(),
+    createEngine(module, host) {
+        const engineHost = host ?? this.aqlHost;
+        let cache;
+        if (engineHost === this._aqlHost) {
+            cache = this.engineCache;
+        }
+        else {
+            cache = this.hostEngineCache.get(engineHost) ?? new Map();
+            this.hostEngineCache.set(engineHost, cache);
+        }
+        const cacheKey = `${this.functionsGeneration}::${this.getExpressionFunctionsCacheKey(module)}`;
+        const cached = cache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const pools = this.getModuleExpressionFunctionsMap(module);
+        const isGridHost = engineHost === this._aqlHost;
+        const engine = createAqlEngine({
+            host: engineHost,
+            scalarFunctions: pools.scalarFunctions ?? {},
+            operatorRegistry: this.getAggregationCatalogue(module),
+            history: isGridHost ? this.getHistoryStore() : undefined,
+            clock: this.getClock(),
+            customObservableNames: [],
         });
-        return aggregationEvaluation;
+        cache.set(cacheKey, engine);
+        return engine;
     }
-    validateBoolean(expressionInput = '', module, config = { force: false }) {
-        const cacheKey = this.getExpressionCacheKey(expressionInput, module);
-        try {
-            if (this.cacheBooleanValidation.has(cacheKey)) {
-                return this.cacheBooleanValidation.get(cacheKey);
+    getAggregationCatalogue(module) {
+        const cacheKey = `${this.functionsGeneration}::${this.getExpressionFunctionsCacheKey(module)}`;
+        const cached = this.aggregationCatalogueCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const pool = this.getModuleExpressionFunctionsMap(module).aggregatedScalarFunctions ?? {};
+        const definitions = this.resolveCustomAggregatedDefinitions();
+        const admitted = {};
+        for (const [name, definition] of Object.entries(definitions)) {
+            if (pool[name]) {
+                admitted[name] = definition;
             }
-            const expression = expressionInput.trim();
-            if (expression === '') {
-                const result = {
-                    isValid: false,
-                    errorMessage: 'empty boolean expression',
-                };
-                this.cacheBooleanValidation.set(cacheKey, result);
-                return result;
+        }
+        const catalogue = createAggregationCatalogue(mapCustomAggregatedFunctionsToOperators(admitted, this.adaptableApi));
+        this.aggregationCatalogueCache.set(cacheKey, catalogue);
+        return catalogue;
+    }
+    resolveCustomAggregatedDefinitions() {
+        const customAggregatedScalarFunctions = this.adaptableApi.optionsApi.getExpressionOptions()?.customAggregatedFunctions;
+        if (!customAggregatedScalarFunctions) {
+            return {};
+        }
+        const raw = typeof customAggregatedScalarFunctions === 'function'
+            ? customAggregatedScalarFunctions({
+                availableExpressionFunctionNames: getTypedKeys(systemAggregatedExpressionFunctions),
+                systemExpressionFunctions: systemAggregatedExpressionFunctions,
+                ...this.adaptableApi.internalApi.buildBaseContext(),
+            }) ?? {}
+            : customAggregatedScalarFunctions;
+        const resolved = {};
+        for (const [name, definition] of Object.entries(raw)) {
+            if (!definition) {
+                continue;
             }
-            const force = config?.force ?? false;
-            if (!this.adaptableApi.optionsApi.getExpressionOptions().performExpressionValidation &&
-                !force) {
-                const result = { isValid: true, errorMessage: '' };
-                this.cacheBooleanValidation.set(cacheKey, result);
-                return result;
+            if (PROTECTED_CORE_EXPRESSION_FUNCTIONS.has(name)) {
+                this.adaptableApi.logWarn(`Custom aggregated function '${name}' attempts to override a structural AQL primitive; ignored`);
+                continue;
             }
-            const { ast } = parser.parse(expression.trim());
-            const rootFn = ast[ast.length - 1];
-            const moduleExpressionFunctions = this.getModuleExpressionFunctionsMap(module);
-            const booleanAndScalarFunctions = this.getBooleanAndScalarFunctions(moduleExpressionFunctions);
-            if (rootFn.type === undefined || booleanAndScalarFunctions[rootFn.type] === undefined) {
-                const result = {
-                    isValid: false,
-                    errorMessage: `provided function ${rootFn?.type} cannot be handled`,
-                };
-                this.cacheBooleanValidation.set(cacheKey, result);
-                return result;
+            if ('extends' in definition) {
+                this.adaptableApi.logWarn(`Custom aggregated function '${name}' uses 'extends', which is not supported for aggregations; ignored`);
+                continue;
             }
-            if (booleanAndScalarFunctions[rootFn.type].returnType !== 'boolean') {
-                const result = {
-                    isValid: false,
-                    errorMessage: `provided function ${rootFn?.type} is not a predicate`,
-                };
-                this.cacheBooleanValidation.set(cacheKey, result);
-                return result;
+            if (typeof definition.reducer !== 'function') {
+                this.adaptableApi.logWarn(`Custom aggregated function '${name}' is missing a reducer and was ignored.`);
+                continue;
             }
-            this.validateCircularQueryReferences(expression, module);
-            const result = {
-                isValid: true,
-                errorMessage: '',
-            };
-            this.cacheBooleanValidation.set(cacheKey, result);
-            return result;
+            if (name in systemAggregatedExpressionFunctions) {
+                this.adaptableApi.logInfo(`Custom aggregated function '${name}' overrides the system aggregation`);
+            }
+            resolved[name] = definition;
         }
-        catch (error) {
-            const result = { isValid: false, errorMessage: error };
-            this.cacheBooleanValidation.set(cacheKey, result);
-            return result;
+        return resolved;
+    }
+    getComputedValueStore() {
+        if (!this._computedValueStore) {
+            this._computedValueStore = createComputedValueStore({
+                engine: {
+                    compile: (spec) => this.createEngine(spec.module).compileValueExpression(spec.expression, { rowSetKind: spec.rowSetSpec }),
+                    executeBatch: (module, compiledExpressions) => this.createEngine(module).executeValueExpressionBatch(compiledExpressions, { ext: this.adaptableApi.internalApi.buildBaseContext() }, {
+                        resolveRowSet: (kind) => {
+                            if (kind === 'visible') {
+                                return this.adaptableApi.gridApi.getVisibleRowNodes();
+                            }
+                            if (kind === 'all') {
+                                return this.adaptableApi.gridApi.getAllRowNodes();
+                            }
+                            return undefined;
+                        },
+                    }),
+                    applyDelta: (module, compiled, previous, changes) => {
+                        const scansEveryRow = compiled.aggregations.every((plan) => plan.scanSignature.rowSetKind === 'all' ||
+                            plan.scanSignature.rowSetKind === DEFAULT_ROW_SET_KIND);
+                        if (!scansEveryRow) {
+                            return undefined;
+                        }
+                        return this.createEngine(module).applyValueExpressionDelta(compiled, previous, changes, { ext: this.adaptableApi.internalApi.buildBaseContext() });
+                    },
+                },
+                batchWindowMs: this.adaptableApi.optionsApi.getCalculatedColumnOptions()
+                    ?.aggregatedColumnRowChangeThrottle ?? 300,
+                clock: this.getClock(),
+                getRowId: (rowRef) => rowRef?.id,
+                history: this.getHistoryStore(),
+                onNodeError: (node, error) => this.adaptableApi.logError(`Computed value node quarantined: "${node.spec.expression}" (${node.spec.module})`, error),
+            });
+            this.bindHostEvents();
         }
+        return this._computedValueStore;
     }
-    validateObservable(expressionInput = '', module) {
-        const cacheKey = this.getExpressionCacheKey(expressionInput, module);
-        if (this.cacheObservableValidation.has(cacheKey)) {
-            return this.cacheObservableValidation.get(cacheKey);
+    bindHostEvents() {
+        if (this.hostEventsBound) {
+            return;
         }
-        const expression = expressionInput.trim();
-        if (expression === '') {
-            const result = {
-                isValid: false,
-                errorMessage: 'empty observable expression',
-            };
-            this.cacheObservableValidation.set(cacheKey, result);
-            return result;
+        const dataService = this.adaptableApi.internalApi.getDataService?.();
+        if (!dataService) {
+            return;
         }
-        try {
-            const observableExpression = this.evaluateObservableExpression(expression, module);
-            if (!isObservable(observableExpression)) {
-                const result = {
-                    isValid: false,
-                    errorMessage: `provided observable expression does not evaluate to an Observable`,
-                };
-                this.cacheObservableValidation.set(cacheKey, result);
-                return result;
+        this.hostEventsBound = true;
+        this.hostEventUnsubscribers.push(dataService.on('BeforeCellDataChanged', (info) => {
+            if (info.trigger === 'aggChange' || info.preventEdit) {
+                return;
+            }
+            const rowNode = info.rowNode;
+            const columnId = info.column?.columnId;
+            if (rowNode && !rowNode.group && columnId !== undefined) {
+                this._historyStore?.recordCellChange({
+                    rowRef: rowNode,
+                    columnId,
+                    oldValue: info.oldValue,
+                    newValue: info.newValue,
+                    trigger: toChangeTrigger(info.trigger),
+                    t: info.changedAt,
+                });
+            }
+        }), dataService.on('BeforeRowDataChanged', (info) => {
+            this._historyStore?.recordRowChange({
+                trigger: info.rowTrigger,
+                rowRefs: info.rowNodes ?? [],
+                t: info.changedAt,
+            });
+        }), dataService.on('CellDataChanged', (info) => {
+            if (info.trigger === 'aggChange' || info.preventEdit) {
+                return;
+            }
+            if (info.trigger === 'calculatedColumnChange') {
+                return;
+            }
+            const rowNode = info.rowNode;
+            const columnId = info.column?.columnId;
+            const store = this._computedValueStore;
+            if (!store || columnId === undefined) {
+                return;
+            }
+            if (rowNode && !rowNode.group) {
+                store.markDirtyForCellChanges([
+                    {
+                        rowRef: rowNode,
+                        columnId,
+                        oldValue: info.oldValue,
+                        newValue: info.newValue,
+                    },
+                ]);
+                return;
+            }
+            store.markDirtyForColumns([columnId]);
+        }), dataService.on('RowDataChanged', (info) => {
+            if (info?.rowTrigger !== 'Update') {
+                this._computedValueStore?.markAllDirty();
+            }
+        }));
+    }
+    getPreparedRowEvaluator(expression, module, host) {
+        const cacheKey = `${this.functionsGeneration}::${this.getExpressionFunctionsCacheKey(module)}::${this.toCacheExpression(expression)}`;
+        const cache = host ? this.getHostRowEvaluatorCache(host) : this.rowEvaluatorCache;
+        const cached = cache.get(cacheKey);
+        if (cached) {
+            if (!host) {
+                cache.delete(cacheKey);
+                cache.set(cacheKey, cached);
+            }
+            return cached;
+        }
+        const evaluator = this.createEngine(module, host).prepareBooleanEvaluator(expression, {
+            ext: this.adaptableApi.internalApi.buildBaseContext(),
+        });
+        if (!host && cache.size >= ROW_EVALUATOR_CACHE_SIZE) {
+            const oldestKey = cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                cache.delete(oldestKey);
+                this.releaseObservableSeries(oldestKey);
             }
-            const result = { isValid: true, errorMessage: '' };
-            this.cacheObservableValidation.set(cacheKey, result);
-            return result;
-        }
-        catch (error) {
-            const result = {
-                isValid: false,
-                errorMessage: error,
-            };
-            this.cacheObservableValidation.set(cacheKey, result);
-            return result;
         }
+        cache.set(cacheKey, evaluator);
+        if (!host) {
+            this.retainObservableSeries(cacheKey, expression, module);
+        }
+        return evaluator;
     }
-    validateAggregatedBoolean(expressionInput = '', module) {
-        const cacheKey = this.getExpressionCacheKey(expressionInput, module);
-        if (this.cacheAggregatedBooleanValidation.has(cacheKey)) {
-            return this.cacheAggregatedBooleanValidation.get(cacheKey);
+    observeExpression(expression, module) {
+        if (!expression?.trim() || !this.getExpressionDependencies(expression, module)?.observable) {
+            return;
         }
-        const expression = expressionInput.trim();
-        if (expression === '') {
-            const result = {
-                isValid: false,
-                errorMessage: 'empty AggregatedBoolean expression',
-            };
-            this.cacheAggregatedBooleanValidation.set(cacheKey, result);
-            return result;
+        try {
+            this.getPreparedRowEvaluator(expression, module);
+        }
+        catch {
+        }
+    }
+    toObservableRetentionKey(evaluatorCacheKey) {
+        return evaluatorCacheKey.slice(evaluatorCacheKey.indexOf('::') + 2);
+    }
+    retainObservableSeries(evaluatorCacheKey, expression, module) {
+        const retentionKey = this.toObservableRetentionKey(evaluatorCacheKey);
+        if (this.observableSeriesReleases.has(retentionKey)) {
+            return;
+        }
+        if (!expression.includes('{') || !this.getExpressionDependencies(expression, module)?.observable) {
+            return;
         }
         try {
-            const evaluationResult = this.evaluateAggregatedBooleanExpression(expression, module);
-            if (evaluationResult.type !== 'aggregationBoolean') {
-                const result = {
-                    isValid: false,
-                    errorMessage: 'provided AggregatedBoolean expression does not evaluate to a supported aggregation',
-                };
-                this.cacheAggregatedBooleanValidation.set(cacheKey, result);
-                return result;
-            }
-            const result = { isValid: true, errorMessage: '' };
-            this.cacheAggregatedBooleanValidation.set(cacheKey, result);
-            return result;
-        }
-        catch (error) {
-            const result = {
-                isValid: false,
-                errorMessage: error,
-            };
-            this.cacheAggregatedBooleanValidation.set(cacheKey, result);
-            return result;
+            const compiled = this.createEngine(module).compileValueExpression(expression);
+            this.observableSeriesReleases.set(retentionKey, acquireObservableSlots(this.getHistoryStore(), compiled.observable));
+        }
+        catch {
         }
     }
-    validateAggregatedScalar(expressionInput = '', module) {
-        const cacheKey = this.getExpressionCacheKey(expressionInput, module);
-        if (this.cacheAggregatedScalarValidation.has(cacheKey)) {
-            return this.cacheAggregatedScalarValidation.get(cacheKey);
+    releaseObservableSeries(evaluatorCacheKey) {
+        const retentionKey = this.toObservableRetentionKey(evaluatorCacheKey);
+        const release = this.observableSeriesReleases.get(retentionKey);
+        if (release) {
+            this.observableSeriesReleases.delete(retentionKey);
+            release();
         }
-        const expression = expressionInput.trim();
-        if (expression === '') {
-            const result = {
-                isValid: false,
-                errorMessage: 'empty AggregatedScalar expression',
-            };
-            this.cacheAggregatedScalarValidation.set(cacheKey, result);
-            return result;
+    }
+    releaseAllObservableSeries() {
+        this.observableSeriesReleases.forEach((release) => release());
+        this.observableSeriesReleases.clear();
+    }
+    getHostRowEvaluatorCache(host) {
+        let cache = this.hostRowEvaluatorCache.get(host);
+        if (!cache) {
+            cache = new Map();
+            this.hostRowEvaluatorCache.set(host, cache);
+        }
+        return cache;
+    }
+    getFirstRowRef(host) {
+        return hostGetFirstRowRef(host) ?? this.adaptableApi.gridApi.getFirstRowNode();
+    }
+    evaluateBooleanExpression(expression, module, rowNode, evalContext) {
+        if (expression == undefined) {
+            this.adaptableApi.logError('QueryLanguageService.evaluateBooleanExpression was called with an undefined expression');
+            return false;
+        }
+        return this.getPreparedRowEvaluator(expression, module, evalContext?.host)(rowNode, this.toPerRowConfig({
+            rowNode,
+            dataChangedEvent: evalContext?.dataChangedEvent,
+            pivotResultColumn: evalContext?.pivotResultColumn,
+            columnScope: evalContext?.columnScope,
+            cellValueOverride: evalContext?.cellValueOverride,
+        }));
+    }
+    evaluateScalarExpression(expression, module, rowNode, evalContext) {
+        return this.evaluateBooleanExpression(expression, module, rowNode, evalContext);
+    }
+    prepareBooleanRowEvaluator(expression, module) {
+        const evaluator = this.getPreparedRowEvaluator(expression, module);
+        return (rowNode) => evaluator(rowNode);
+    }
+    executeValueExpression(expression, module, evalContext) {
+        return this.createEngine(module, evalContext?.host).executeValueExpression(expression, this.toEvaluationConfig({
+            rowNode: evalContext?.rowNode ?? this.getFirstRowRef(evalContext?.host),
+            cellValueOverride: evalContext?.cellValueOverride,
+            dataChangedEvent: evalContext?.dataChangedEvent,
+            columnScope: evalContext?.columnScope,
+            getRowNodes: evalContext?.getRowNodes,
+        }), { getValue: evalContext?.getColumnValue });
+    }
+    getExpressionDependencies(expression, module) {
+        return this.getCompiledExpressionSummary(expression, module)?.dependencies;
+    }
+    getExpressionOutputType(expression, module) {
+        return this.getCompiledExpressionSummary(expression, module)?.outputType;
+    }
+    isExpressionRowIndependent(expression, module) {
+        return this.getCompiledExpressionSummary(expression, module)?.isRowIndependent ?? false;
+    }
+    getCompiledExpressionSummary(expression, module) {
+        if (!expression?.trim()) {
+            return undefined;
+        }
+        const cacheKey = `${this.functionsGeneration}::${this.getExpressionFunctionsCacheKey(module)}::${this.toCacheExpression(expression)}`;
+        const cached = this.dependenciesCache.get(cacheKey);
+        if (cached !== undefined) {
+            return cached ?? undefined;
         }
+        let summary;
         try {
-            const evaluationResult = this.evaluateAggregatedScalarExpression(expression, module);
-            if (evaluationResult.type !== 'aggregationScalar') {
-                const result = {
-                    isValid: false,
-                    errorMessage: 'provided AggregatedScalar expression does not evaluate to a supported aggregation',
-                };
-                this.cacheAggregatedScalarValidation.set(cacheKey, result);
-                return result;
-            }
-            const result = { isValid: true, errorMessage: '' };
-            this.cacheAggregatedScalarValidation.set(cacheKey, result);
-            return result;
-        }
-        catch (error) {
-            const result = {
-                isValid: false,
-                errorMessage: error,
+            const compiled = this.createEngine(module).compileValueExpression(expression);
+            summary = {
+                dependencies: compiled.dependencies,
+                outputType: compiled.outputType,
+                isRowIndependent: compiled.isRowIndependent,
             };
-            this.cacheAggregatedScalarValidation.set(cacheKey, result);
-            return result;
-        }
-    }
-    computeAggregatedBooleanValue(expression, module) {
-        const booleanAggregationParameter = this.evaluateAggregatedBooleanExpression(expression, module);
-        const aggregatedScalarExpressionEvaluation = booleanAggregationParameter.scalarAggregation.value;
-        const aggregatedScalarLiveValue = new AggregatedScalarLiveValue({ aggregatedScalarExpressionEvaluation }, module, this.adaptableApi);
-        const allAggregationValues = aggregatedScalarLiveValue.getAllAggregationValues();
-        const numericOperand = booleanAggregationParameter.conditionValue;
-        const booleanConditionFn = booleanAggregationParameter.conditionFn;
-        return allAggregationValues.some((aggregationValue) => {
-            return booleanConditionFn(aggregationValue, numericOperand);
+        }
+        catch {
+            summary = null;
+        }
+        this.dependenciesCache.set(cacheKey, summary);
+        return summary ?? undefined;
+    }
+    isAggregatedBooleanSatisfied(compiled, result, module, evalContext) {
+        return this.createEngine(module, evalContext?.host).isAggregatedBooleanSatisfied(compiled, result);
+    }
+    rowMatchesCompiledFilters(compiled, rowNode, module, evalContext) {
+        return this.createEngine(module, evalContext?.host).rowMatchesCompiledFilters(compiled, rowNode, {
+            cellValueOverride: evalContext?.cellValueOverride,
+            ext: this.adaptableApi.internalApi.buildBaseContext(),
         });
     }
-    getColumnsFromExpression(input = '') {
-        return this.getNodesFromExpression(input, 'COL');
+    computeAggregatedExpressionValue(expression, module, getRowNodes, evalContext) {
+        const booleanValidation = this.validateAggregatedBoolean(expression, module, {
+            host: evalContext?.host,
+        });
+        if (booleanValidation.isValid) {
+            const engine = this.createEngine(module, evalContext?.host);
+            const compiled = engine.compileValueExpression(expression);
+            const result = engine.executeValueExpression(compiled, this.toEvaluationConfig({
+                rowNode: this.getFirstRowRef(evalContext?.host),
+                getRowNodes,
+            }), { getValue: evalContext?.getColumnValue });
+            return this.toAggregatedSnapshot(result, engine.isAggregatedBooleanSatisfied(compiled, result));
+        }
+        return this.computeAggregatedScalarValue(expression, module, getRowNodes, evalContext);
     }
-    getNamedQueryNamesFromExpression(input = '') {
-        return this.getNodesFromExpression(input, 'QUERY');
+    computeAggregatedScalarValue(expression, module, getRowNodes, evalContext) {
+        const result = this.executeValueExpression(expression, module, {
+            host: evalContext?.host,
+            getRowNodes,
+            getColumnValue: evalContext?.getColumnValue,
+        });
+        return this.toAggregatedSnapshot(result);
     }
-    getExpressionWithColumnFriendlyNames(expression = '') {
-        let result = expression;
-        const columnIds = this.getColumnsFromExpression(expression);
-        columnIds.forEach((columnId) => {
-            const columnFriendlyName = this.adaptableApi.columnApi.getFriendlyNameForColumnId(columnId);
-            result = result.split(columnId).join(columnFriendlyName);
+    toAggregatedSnapshot(result, matches) {
+        const aggregation = result.aggregations[0];
+        return {
+            value: (aggregation?.value ?? result.value),
+            groupValues: aggregation?.groupValues,
+            getValueForRow: aggregation?.getValueForRow ?? result.getValueForRow,
+            rowCount: result.rowCount,
+            ...(matches !== undefined ? { matches } : {}),
+        };
+    }
+    computeAggregatedBooleanValue(expression, module, getRowNodes, evalContext) {
+        const engine = this.createEngine(module, evalContext?.host);
+        const compiled = engine.compileValueExpression(expression);
+        const result = engine.executeValueExpression(compiled, this.toEvaluationConfig({
+            rowNode: this.getFirstRowRef(evalContext?.host),
+            getRowNodes,
+        }), { getValue: evalContext?.getColumnValue });
+        return engine.isAggregatedBooleanSatisfied(compiled, result);
+    }
+    evaluateQueryVariable(functionName, args) {
+        const queryVariable = this.adaptableApi.optionsApi
+            .getExpressionOptions()
+            ?.queryVariables?.find((variable) => variable.name === functionName);
+        if (!queryVariable) {
+            return undefined;
+        }
+        const context = {
+            args,
+            ...this.adaptableApi.internalApi.buildBaseContext(),
+        };
+        return typeof queryVariable.value === 'function'
+            ? queryVariable.value(context)
+            : queryVariable.value;
+    }
+    getExpressionKind(expressionInput = '', module, fallback = 'scalar') {
+        const expression = typeof expressionInput === 'string' ? expressionInput.trim() : '';
+        if (expression === '') {
+            return fallback;
+        }
+        const cacheKey = `${this.functionsGeneration}::${module}::${expression}`;
+        const cached = this.expressionKindCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        const kind = inferExpressionKindFromText(expression, {
+            customAggregatedNames: this.getAggregationCatalogue(module).customNames(),
         });
-        result = result.split(`$SCOPE()`).join(`$Column_Scope`);
-        return result;
+        if (kind === undefined) {
+            return fallback;
+        }
+        this.expressionKindCache.set(cacheKey, kind);
+        return kind;
     }
-    getModuleExpressionFunctionsMap(module) {
-        const expressionOptions = this.adaptableApi.optionsApi.getExpressionOptions();
-        if (module) {
-            let cachedResult = this.cacheModuleSpecificExpressionFunctions.get(module);
-            if (cachedResult) {
-                return cachedResult;
+    resolveExpression(expression, module, expectedKind) {
+        const text = typeof expression === 'string' ? expression : '';
+        return { kind: this.getExpressionKind(text, module, expectedKind), text };
+    }
+    validateBoolean(expressionInput = '', module, config = { force: false }) {
+        return this.validateScalarLike(expressionInput, module, {
+            kind: 'boolean',
+            emptyMessage: 'empty boolean expression',
+            force: config?.force,
+            requireBooleanRoot: true,
+        });
+    }
+    validateScalar(expressionInput = '', module, config = { force: false }) {
+        return this.validateScalarLike(expressionInput, module, {
+            kind: 'scalar',
+            emptyMessage: 'empty scalar expression',
+            force: config?.force,
+        });
+    }
+    validateAggregatedBoolean(expressionInput = '', module, evalContext) {
+        return this.validateAggregatedKind(expressionInput, module, {
+            kind: 'aggregatedBoolean',
+            emptyMessage: 'empty AggregatedBoolean expression',
+            mismatchMessage: 'provided AggregatedBoolean expression does not evaluate to a supported aggregation',
+            host: evalContext?.host,
+        });
+    }
+    validateAggregatedScalar(expressionInput = '', module, evalContext) {
+        return this.validateAggregatedKind(expressionInput, module, {
+            kind: 'aggregatedScalar',
+            emptyMessage: 'empty AggregatedScalar expression',
+            mismatchMessage: 'provided AggregatedScalar expression does not evaluate to a supported aggregation',
+            host: evalContext?.host,
+        });
+    }
+    validateScalarLike(expressionInput, module, options) {
+        return this.withValidationCache(options.kind, expressionInput, module, () => {
+            const expression = expressionInput.trim();
+            if (expression === '') {
+                return invalid(options.emptyMessage);
+            }
+            if (!this.adaptableApi.optionsApi.getExpressionOptions().performExpressionValidation &&
+                !options.force) {
+                return VALID;
+            }
+            try {
+                const { ast } = parse(expression);
+                const rootFn = ast[ast.length - 1];
+                const engine = this.createEngine(module);
+                const scalarFunctions = this.getModuleExpressionFunctionsMap(module).scalarFunctions;
+                if (rootFn.type === undefined || scalarFunctions[rootFn.type] === undefined) {
+                    return invalid(`provided function ${rootFn?.type} cannot be handled`);
+                }
+                if (options.requireBooleanRoot) {
+                    const booleanRootError = engine.getBooleanRootError(rootFn);
+                    if (booleanRootError) {
+                        return invalid(booleanRootError);
+                    }
+                }
+                this.validateCircularQueryReferences(expression);
+                const typeCheck = validateExpressionTypes(ast, scalarFunctions, {
+                    getBraceCallSpec: engine.aggregationCatalogue.getBraceCallSpec,
+                });
+                if (!typeCheck.isValid) {
+                    return invalid(typeCheck.issues[0]?.message ?? 'invalid expression types');
+                }
+                return this.validateExpressionStructure(expression, module, options.kind) ?? VALID;
+            }
+            catch (error) {
+                return invalid(error);
+            }
+        });
+    }
+    validateAggregatedKind(expressionInput, module, options) {
+        return this.withValidationCache(options.kind, expressionInput, module, () => {
+            const expression = expressionInput.trim();
+            if (expression === '') {
+                return invalid(options.emptyMessage);
             }
+            const structuralError = this.validateExpressionStructure(expression, module, options.kind);
+            if (structuralError) {
+                return structuralError;
+            }
+            try {
+                const compiled = this.createEngine(module, options.host).compileValueExpression(expression);
+                if (compiled.aggregations.length === 0) {
+                    return invalid(options.mismatchMessage);
+                }
+                if (options.kind === 'aggregatedBoolean' && compiled.outputType !== 'boolean') {
+                    return invalid(options.mismatchMessage);
+                }
+                if (options.kind === 'aggregatedScalar' && compiled.outputType === 'boolean') {
+                    return invalid(options.mismatchMessage);
+                }
+                return VALID;
+            }
+            catch (error) {
+                return invalid(error);
+            }
+        }, !options.host);
+    }
+    validateExpressionStructure(expression, module, kind) {
+        let ast;
+        try {
+            ast = parse(expression).ast;
         }
-        const generalBooleanExpressionFunctions = this.extractMappedExpressionFunctions(booleanExpressionFunctions, expressionOptions.systemBooleanFunctions, expressionOptions.customBooleanFunctions);
-        const generalScalarExpressionFunctions = this.extractMappedExpressionFunctions(scalarExpressionFunctions, expressionOptions.systemScalarFunctions, expressionOptions.customScalarFunctions);
-        const generalObservableExpressionFunctions = this.extractMappedExpressionFunctions(observableExpressionFunctions, expressionOptions.systemObservableFunctions);
-        const generalAggregatedBooleanExpressionFunctions = this.extractMappedExpressionFunctions(aggregatedBooleanExpressionFunctions, expressionOptions.systemAggregatedBooleanFunctions, this.getCustomAggregatedScalarFunctions());
-        const generalAggregatedScalarExpressionFunctions = this.extractMappedExpressionFunctions(aggregatedScalarExpressionFunctions, expressionOptions.systemAggregatedScalarFunctions, this.getCustomAggregatedScalarFunctions());
+        catch {
+            return undefined;
+        }
+        const pools = this.getModuleExpressionFunctionsMap(module);
+        const expressionOptions = this.adaptableApi.optionsApi.getExpressionOptions();
+        const result = validateAqlExpression(ast, {
+            request: {
+                ...mapLegacyKindToValidationRequest(kind),
+                requiresAggCalls: false,
+                allowsObservableCalls: moduleAllowsObservableCalls(module),
+            },
+            scalarFunctions: pools.scalarFunctions,
+            catalogue: this.getAggregationCatalogue(module),
+            customObservableNames: getCustomPoolNames(pools.observableFunctions, systemObservableExpressionFunctions),
+            maxWithinMs: expressionOptions.maxTimeframeSize,
+            maxHistoryPerCell: expressionOptions.maxHistoryPerCell,
+            getColumnType: (columnId) => mapDataTypeToStaticType(this.adaptableApi.columnApi.getColumnWithColumnId(columnId, false)?.dataType),
+        });
+        const firstError = result.issues.find((issue) => issue.severity === 'error');
+        return firstError ? invalid(firstError.message) : undefined;
+    }
+    withValidationCache(kind, expression, module, compute, useCache = true) {
+        const cacheKey = `${kind}::${module}::${this.toCacheExpression(expression)}`;
+        if (useCache && this.validationCache.has(cacheKey)) {
+            return this.validationCache.get(cacheKey);
+        }
+        const result = compute();
+        if (useCache) {
+            this.validationCache.set(cacheKey, result);
+        }
+        return result;
+    }
+    getModuleExpressionFunctionsMap(module) {
         if (!module) {
             this.adaptableApi.logWarn(`QueryLanguageService.getModuleExpressionFunctions() was called with an undefined 'module' param, this should never happen`);
-            return {
-                booleanFunctions: generalBooleanExpressionFunctions,
-                scalarFunctions: generalScalarExpressionFunctions,
-                observableFunctions: generalObservableExpressionFunctions,
-                aggregatedBooleanFunctions: generalAggregatedBooleanExpressionFunctions,
-                aggregatedScalarFunctions: generalAggregatedScalarExpressionFunctions,
-            };
-        }
-        let moduleExpressionFunctions;
-        if (typeof expressionOptions.moduleExpressionFunctions === 'function') {
-            const context = {
-                module,
-                availableBooleanFunctionNames: getTypedKeys(generalBooleanExpressionFunctions),
-                availableScalarFunctionNames: getTypedKeys(generalScalarExpressionFunctions),
-                availableObservableFunctionNames: getTypedKeys(generalObservableExpressionFunctions),
-                availableAggregatedBooleanFunctionNames: getTypedKeys(generalAggregatedBooleanExpressionFunctions),
-                availableAggregatedScalarFunctionNames: getTypedKeys(generalAggregatedScalarExpressionFunctions),
-                ...this.adaptableApi.internalApi.buildBaseContext(),
-            };
-            moduleExpressionFunctions = expressionOptions.moduleExpressionFunctions(context) ?? {};
+            return this.resolveExpressionFunctionsMap(module);
         }
-        else {
-            moduleExpressionFunctions = expressionOptions.moduleExpressionFunctions?.[module] ?? {};
-        }
-        const moduleExpressionFunctionsMap = {
-            booleanFunctions: this.extractMappedExpressionFunctions(generalBooleanExpressionFunctions, moduleExpressionFunctions.systemBooleanFunctions, moduleExpressionFunctions.customBooleanFunctions),
-            scalarFunctions: this.extractMappedExpressionFunctions(generalScalarExpressionFunctions, moduleExpressionFunctions.systemScalarFunctions, moduleExpressionFunctions.customScalarFunctions),
-            observableFunctions: this.extractMappedExpressionFunctions(generalObservableExpressionFunctions, moduleExpressionFunctions.systemObservableFunctions),
-            aggregatedBooleanFunctions: this.extractMappedExpressionFunctions(generalAggregatedBooleanExpressionFunctions, moduleExpressionFunctions.systemAggregatedBooleanFunctions),
-            aggregatedScalarFunctions: this.extractMappedExpressionFunctions(generalAggregatedScalarExpressionFunctions, moduleExpressionFunctions.systemAggregatedScalarFunctions),
-        };
-        this.cacheModuleSpecificExpressionFunctions.set(module, moduleExpressionFunctionsMap);
-        return moduleExpressionFunctionsMap;
+        const cached = this.moduleFunctionsCache.get(this.getExpressionFunctionsCacheKey(module));
+        return cached ?? this.resolveModuleExpressionFunctionsMap(module);
     }
-    getCustomAggregatedScalarFunctions() {
-        const customAggregatedScalarFunctions = this.adaptableApi.optionsApi.getExpressionOptions()?.customAggregatedFunctions;
-        if (!customAggregatedScalarFunctions) {
-            return {};
+    resolveModuleExpressionFunctionsMap(module) {
+        const cacheKey = module ? this.getExpressionFunctionsCacheKey(module) : undefined;
+        const refreshing = cacheKey != null && this.moduleFunctionsCache.has(cacheKey);
+        const resolved = this.resolveExpressionFunctionsMap(module);
+        if (cacheKey) {
+            this.moduleFunctionsCache.set(cacheKey, resolved);
         }
-        if (typeof customAggregatedScalarFunctions === 'function') {
-            return (context) => {
-                const customDefinitions = customAggregatedScalarFunctions(context);
-                return this.mapCustomAggregatedScalarFunctionsToInternal(customDefinitions);
-            };
+        if (refreshing) {
+            this.functionsGeneration += 1;
+            this.rowEvaluatorCache.clear();
+            this.dependenciesCache.clear();
+            this.validationCache.clear();
+            this.expressionKindCache.clear();
+            this.aggregationCatalogueCache.clear();
+            this.engineCache.clear();
         }
-        return this.mapCustomAggregatedScalarFunctionsToInternal(customAggregatedScalarFunctions);
+        return resolved;
     }
-    mapCustomAggregatedScalarFunctionsToInternal(aggregatedScalarFunctions) {
-        const getValueForColId = (colId, rowNode) => {
-            return this.adaptableApi.gridApi.getRawValueFromRowNode(rowNode, colId);
+    getExpressionFunctionsCacheKey(module) {
+        return `${module}::${this.adaptableApi.optionsApi.getUserName()}`;
+    }
+    resolveExpressionFunctionsMap(module) {
+        const expressionOptions = this.adaptableApi.optionsApi.getExpressionOptions();
+        const customStandardFunctions = this.resolveCustomFunctions(expressionOptions.customStandardFunctions, systemScalarExpressionFunctions);
+        const customAggregatedFunctions = Object.fromEntries(Object.entries(this.resolveCustomAggregatedDefinitions()).map(([name, definition]) => [
+            name,
+            toCustomAggregationFunctionDef(definition),
+        ]));
+        const apply = (pool, functionType, customNames) => applyExpressionFunctionAvailability(pool, {
+            functionType,
+            module,
+            customNames,
+            globalOverride: expressionOptions.isExpressionFunctionAvailable,
+            buildBaseContext: () => this.adaptableApi.internalApi.buildBaseContext(),
+            logWarn: (message) => this.adaptableApi.logWarn(message),
+        });
+        const observableFunctions = apply({ ...systemObservableExpressionFunctions }, 'observable', new Set());
+        return {
+            scalarFunctions: {
+                ...observableFunctions,
+                ...apply({ ...systemScalarExpressionFunctions, ...customStandardFunctions }, 'scalar', new Set(Object.keys(customStandardFunctions))),
+            },
+            observableFunctions,
+            aggregatedScalarFunctions: apply({
+                ...aggregationLookupFunctions,
+                ...systemAggregationClauseFunctions,
+                ...systemAggregatedExpressionFunctions,
+                ...customAggregatedFunctions,
+            }, 'aggregated', new Set(Object.keys(customAggregatedFunctions))),
         };
-        return Object.entries(aggregatedScalarFunctions).reduce((acc, [expressionName, customExpressionDefinition]) => {
-            const expression = {
-                handler: (args, context) => {
-                    const [aggColParam, ...restOfColParams] = extractColumnParameters(expressionName, args);
-                    const aggColumnId = aggColParam?.value;
-                    const groupByParameter = extractParameter(expressionName, 'operand', ['GROUP_BY'], args, {
-                        isOptional: true,
-                    });
-                    const groupByColumnIds = groupByParameter?.value;
-                    const contextArgs = args.filter((arg) => {
-                        if (typeof arg === 'object' && arg.name === 'COL') {
-                            return arg.value !== aggColumnId;
-                        }
-                        if (typeof arg === 'object' && arg.name === 'GROUP_BY') {
-                            return false;
-                        }
-                        return true;
-                    });
-                    const aggregationExpressionEvaluation = {
-                        aggregationParams: {
-                            reducers: {
-                                [expressionName]: {
-                                    name: expressionName,
-                                    field: aggColumnId,
-                                    initialValue: customExpressionDefinition.initialValue,
-                                    reducer: (acc, value, rowNode, dataIndex) => {
-                                        const context = {
-                                            accumulator: acc,
-                                            currentValue: value,
-                                            index: dataIndex,
-                                            rowNode,
-                                            args: contextArgs.map((arg) => {
-                                                if (typeof arg === 'object' && arg.name === 'COL') {
-                                                    return getValueForColId(arg.value, rowNode);
-                                                }
-                                                return arg;
-                                            }),
-                                            aggColumnId,
-                                            groupByColumnIds,
-                                            getValueForColId: (colId) => getValueForColId(colId, rowNode),
-                                            ...this.adaptableApi.internalApi.buildBaseContext(),
-                                        };
-                                        return customExpressionDefinition.reducer(context);
-                                    },
-                                    done: (accValue, dataArray) => {
-                                        if (customExpressionDefinition.processAggregatedValue) {
-                                            const context = {
-                                                ...this.adaptableApi.internalApi.buildBaseContext(),
-                                                aggregatedValue: accValue,
-                                                rowNodes: dataArray,
-                                                args: contextArgs,
-                                                aggColumnId,
-                                                groupByColumnIds,
-                                            };
-                                            return customExpressionDefinition.processAggregatedValue(context);
-                                        }
-                                        return accValue;
-                                    },
-                                },
-                            },
-                        },
-                        rowValueGetter: (rowNode, aggregationValue) => {
-                            if (customExpressionDefinition.prepareRowValue) {
-                                const context = {
-                                    rowNode: rowNode,
-                                    aggregatedValue: aggregationValue,
-                                    ...this.adaptableApi.internalApi.buildBaseContext(),
-                                    args: contextArgs.map((arg) => {
-                                        if (typeof arg === 'object' && arg.name === 'COL') {
-                                            return getValueForColId(arg.value, rowNode);
-                                        }
-                                        return arg;
-                                    }),
-                                    aggColumnId,
-                                    groupByColumnIds,
-                                    getValueForColId: (colId) => getValueForColId(colId, rowNode),
-                                };
-                                return customExpressionDefinition.prepareRowValue(context);
-                            }
-                            return aggregationValue;
-                        },
-                        rowFilterFn: (rowNode) => {
-                            if (customExpressionDefinition.filterRow) {
-                                return customExpressionDefinition.filterRow({
-                                    rowNode,
-                                    ...this.adaptableApi.internalApi.buildBaseContext(),
-                                    args: contextArgs.map((arg) => {
-                                        if (typeof arg === 'object' && arg.name === 'COL') {
-                                            return getValueForColId(arg.value, rowNode);
-                                        }
-                                        return arg;
-                                    }),
-                                    aggColumnId,
-                                    groupByColumnIds,
-                                    getValueForColId: (colId) => getValueForColId(colId, rowNode),
-                                });
-                            }
-                            return context?.filterFn?.(rowNode) ?? true;
-                        },
-                        getRowNodes: context.getRowNodes,
-                    };
-                    addGroupByParams(groupByParameter?.value, aggregationExpressionEvaluation);
-                    const result = {
-                        name: expressionName,
-                        type: 'aggregationScalar',
-                        value: aggregationExpressionEvaluation,
-                    };
-                    return result;
-                },
-                description: customExpressionDefinition.description,
-                signatures: customExpressionDefinition.signatures,
-                inputs: customExpressionDefinition.inputTypes,
-            };
-            acc[expressionName] = expression;
-            return acc;
-        }, {});
-    }
-    extractMappedExpressionFunctions(availableExpressionFunctions, systemFunctions, customFunctions) {
-        const systemFunctionNames = typeof systemFunctions === 'function'
-            ? systemFunctions({
-                availableExpressionFunctionNames: getTypedKeys(availableExpressionFunctions),
-                ...this.adaptableApi.internalApi.buildBaseContext(),
-            })
-            : systemFunctions;
-        let generalExpressionFunctions = {};
-        if (Array.isArray(systemFunctionNames)) {
-            systemFunctionNames.forEach((systemFunctionName) => {
-                generalExpressionFunctions[systemFunctionName] =
-                    availableExpressionFunctions[systemFunctionName];
-            });
-        }
-        else {
-            generalExpressionFunctions = { ...availableExpressionFunctions };
+    }
+    resolveCustomFunctions(customFunctions, systemPool) {
+        if (!customFunctions) {
+            return {};
         }
-        const customFunctionDefinitions = typeof customFunctions === 'function'
+        const rawCustom = typeof customFunctions === 'function'
             ? customFunctions({
-                availableExpressionFunctionNames: getTypedKeys(generalExpressionFunctions),
+                availableExpressionFunctionNames: getTypedKeys(systemPool),
+                systemExpressionFunctions: systemPool,
                 ...this.adaptableApi.internalApi.buildBaseContext(),
-            })
+            }) ?? {}
             : customFunctions;
-        if (customFunctionDefinitions) {
-            generalExpressionFunctions = { ...generalExpressionFunctions, ...customFunctionDefinitions };
-        }
-        return generalExpressionFunctions;
+        return resolveCustomExpressionFunctions(rawCustom, systemPool, {
+            warn: (message) => this.adaptableApi.logWarn(message),
+            info: (message) => this.adaptableApi.logInfo(message),
+        });
     }
-    evaluateCustomQueryVariable(functionName, args) {
-        const context = {
-            args,
-            ...this.adaptableApi.internalApi.buildBaseContext(),
-        };
-        const customQueryVariableDefinition = this.adaptableApi.optionsApi.getExpressionOptions()?.customQueryVariables?.[functionName];
-        return typeof customQueryVariableDefinition === 'function'
-            ? customQueryVariableDefinition(context)
-            : customQueryVariableDefinition;
+    getColumnsFromExpression(input = '') {
+        return this.getNodesFromExpression(input, 'COL');
     }
-    getBooleanAndScalarFunctions(moduleExpressionFunctionsMap) {
-        return {
-            ...moduleExpressionFunctionsMap.booleanFunctions,
-            ...moduleExpressionFunctionsMap.scalarFunctions,
-        };
+    getNamedQueryNamesFromExpression(input = '') {
+        return this.getNodesFromExpression(input, 'QUERY');
     }
-    getExpressionCacheKey(expression, module) {
-        return `${module}::${expression}`;
+    getExpressionWithColumnFriendlyNames(expression = '') {
+        let result = expression;
+        for (const columnId of this.getColumnsFromExpression(expression)) {
+            const columnFriendlyName = this.adaptableApi.columnApi.getFriendlyNameForColumnId(columnId);
+            result = result.split(columnId).join(columnFriendlyName);
+        }
+        result = result.replace(/\$SCOPE\s*\(\s*\)/g, '$Column_Scope');
+        result = result.replace(/\$SCOPE\b/g, '$Column_Scope');
+        return result;
     }
     getNodesFromExpression(input, nodeType) {
-        try {
-            const resultSet = new Set();
-            const walk = (node) => {
-                if (typeof node !== 'object') {
-                    return false;
-                }
-                if (Array.isArray(node)) {
-                    return node.map(walk);
-                }
-                node.args.map(walk);
-                if (node.type === nodeType) {
-                    resultSet.add(String(node.args[0]));
-                }
-            };
-            const { ast } = parser.parse(input.trim());
-            walk(ast);
-            return Array.from(resultSet.values());
-        }
-        catch (e) {
-            return [];
-        }
+        return getNodesFromExpression(input, nodeType);
     }
-    validateCircularQueryReferences(expression, module) {
+    validateCircularQueryReferences(expression) {
         if (!expression || expression.trim() === '') {
             return;
         }
@@ -527,9 +799,36 @@ export class QueryLanguageService {
         if (!queryNodes.length) {
             return;
         }
-        const firstRowNode = this.adaptableApi.gridApi.getFirstRowNode();
-        queryNodes.forEach((queryNode) => {
-            this.evaluateBooleanExpression(`QUERY("${queryNode}")`, module, firstRowNode);
-        });
+        const getReferencedQueryNames = (queryName) => {
+            const namedQuery = this.adaptableApi.namedQueryApi.getNamedQueryByName(queryName);
+            if (!namedQuery) {
+                throw new ExpressionEvaluationError('QUERY', `Named Query with name ${queryName} not found!`);
+            }
+            return this.getNamedQueryNamesFromExpression(namedQuery.Expression);
+        };
+        for (const queryName of queryNodes) {
+            throwIfCycle(findDependencyCycle(queryName, getReferencedQueryNames));
+        }
+    }
+    validateCircularCalculatedColumnReferences(expression, calculatedColumnId) {
+        if (!expression || expression.trim() === '' || !calculatedColumnId) {
+            return;
+        }
+        const calculatedColumnApi = this.adaptableApi.calculatedColumnApi;
+        const isCalculatedColumnNode = (columnId) => columnId === calculatedColumnId ||
+            !!calculatedColumnApi.getCalculatedColumnForColumnId(columnId);
+        const getExpressionForColumn = (columnId) => {
+            if (columnId === calculatedColumnId) {
+                return expression;
+            }
+            return calculatedColumnApi.getCalculatedColumnForColumnId(columnId)?.Expression;
+        };
+        const getReferencedCalculatedColumnIds = (columnId) => {
+            const columnExpression = getExpressionForColumn(columnId);
+            return columnExpression
+                ? this.getColumnsFromExpression(columnExpression).filter(isCalculatedColumnNode)
+                : [];
+        };
+        throwIfCycle(findDependencyCycle(calculatedColumnId, getReferencedCalculatedColumnIds));
     }
 }