@cubejs-backend/query-orchestrator 0.30.43 → 0.30.47

package/dist/src/orchestrator/PreAggregations.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PreAggregations = exports.PreAggregationPartitionRangeLoader = exports.PreAggregationLoader = exports.getLastUpdatedAtTimestamp = void 0;
+exports.PreAggregations = exports.PreAggregationPartitionRangeLoader = exports.PreAggregationLoader = exports.getLastUpdatedAtTimestamp = exports.LAMBDA_TABLE_PREFIX = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const ramda_1 = __importDefault(require("ramda"));
 const shared_1 = require("@cubejs-backend/shared");
@@ -13,6 +13,8 @@ const LocalCacheDriver_1 = require("./LocalCacheDriver");
 const QueryCache_1 = require("./QueryCache");
 const ContinueWaitError_1 = require("./ContinueWaitError");
 const StreamObjectsCounter_1 = require("./StreamObjectsCounter");
+/// Name of the inline table containing the lambda rows.
+exports.LAMBDA_TABLE_PREFIX = 'lambda';
 function encodeTimeStamp(time) {
     return Math.floor(time / 1000).toString(32);
 }
@@ -71,7 +73,8 @@ const tablesToVersionEntries = (schema, tables) => ramda_1.default.sortBy(table
     const entity = {
         table_name: `${schema}.${match[1]}`,
         content_version: match[2],
-        structure_version: match[3]
+        structure_version: match[3],
+        build_range_end: table.build_range_end,
     };
     if (match[4].length < 13) {
         entity.last_updated_at = decodeTimeStamp(match[4]);
@@ -129,9 +132,13 @@ class PreAggregationLoadCache {
     async getTablesQuery(preAggregation) {
         const redisKey = this.tablesRedisKey(preAggregation);
         if (!this.tables[redisKey]) {
-            this.tables[redisKey] = this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external ?
+            const tables = this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external ?
                 await this.fetchTablesNoCache(preAggregation) :
                 await this.tablesFromCache(preAggregation);
+            if (tables === undefined) {
+                throw new Error('Pre-aggregation tables are undefined.');
+            }
+            this.tables[redisKey] = tables;
         }
         return this.tables[redisKey];
     }
@@ -300,7 +307,8 @@ class PreAggregationLoader {
                 return {
                     targetTableName: this.targetTableName(versionEntryByStructureVersion),
                     refreshKeyValues: [],
-                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at
+                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
+                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
                 };
             }
             else {
@@ -329,7 +337,8 @@ class PreAggregationLoader {
             return {
                 targetTableName: this.targetTableName(versionEntryByContentVersion),
                 refreshKeyValues: [],
-                lastUpdatedAt: versionEntryByContentVersion.last_updated_at
+                lastUpdatedAt: versionEntryByContentVersion.last_updated_at,
+                buildRangeEnd: versionEntryByContentVersion.build_range_end,
             };
         }
         // TODO this check can be redundant due to structure version is already checked in loadPreAggregation()
@@ -341,7 +350,8 @@ class PreAggregationLoader {
                 return {
                     targetTableName: this.targetTableName(versionEntryByStructureVersion),
                     refreshKeyValues: [],
-                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at
+                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
+                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
                 };
             }
         }
@@ -370,7 +380,8 @@ class PreAggregationLoader {
             return {
                 targetTableName: this.targetTableName(lastVersion),
                 refreshKeyValues: [],
-                lastUpdatedAt: lastVersion.last_updated_at
+                lastUpdatedAt: lastVersion.last_updated_at,
+                buildRangeEnd: lastVersion.build_range_end,
             };
         };
         if (this.forceBuild) {
@@ -424,7 +435,8 @@ class PreAggregationLoader {
         return {
             targetTableName: this.targetTableName(versionEntry),
             refreshKeyValues: [],
-            lastUpdatedAt: versionEntry.last_updated_at
+            lastUpdatedAt: versionEntry.last_updated_at,
+            buildRangeEnd: versionEntry.build_range_end,
         };
     }
     contentVersion(invalidationKeys) {
@@ -464,7 +476,7 @@ class PreAggregationLoader {
             invalidationKeys,
             forceBuild: this.forceBuild,
             metadata: this.metadata,
-            orphanedTimeout: this.orphanedTimeout
+            orphanedTimeout: this.orphanedTimeout,
         }, priority, 
         // eslint-disable-next-line no-use-before-define
         { stageQueryKey: PreAggregations.preAggregationQueryCacheKey(this.preAggregation), requestId: this.requestId });
@@ -478,18 +490,16 @@ class PreAggregationLoader {
         // eslint-disable-next-line no-use-before-define
         return PreAggregations.targetTableName(versionEntry);
     }
-    refresh(preAggregation, newVersionEntry, invalidationKeys) {
-        return (client) => {
-            let refreshStrategy = this.refreshImplStoreInSourceStrategy;
-            if (this.preAggregation.external) {
-                const readOnly = this.preAggregation.readOnly ||
-                    client.config && client.config.readOnly ||
-                    client.readOnly && (typeof client.readOnly === 'boolean' ? client.readOnly : client.readOnly());
-                refreshStrategy = readOnly ?
-                    this.refreshImplStreamExternalStrategy : this.refreshImplTempTableExternalStrategy;
-            }
-            return utils_1.cancelCombinator(saveCancelFn => refreshStrategy.bind(this)(client, newVersionEntry, saveCancelFn, preAggregation, invalidationKeys));
-        };
+    refresh(newVersionEntry, invalidationKeys, client) {
+        let refreshStrategy = this.refreshImplStoreInSourceStrategy;
+        if (this.preAggregation.external) {
+            const readOnly = this.preAggregation.readOnly ||
+                client.config && client.config.readOnly ||
+                client.readOnly && (typeof client.readOnly === 'boolean' ? client.readOnly : client.readOnly());
+            refreshStrategy = readOnly ?
+                this.refreshImplStreamExternalStrategy : this.refreshImplTempTableExternalStrategy;
+        }
+        return utils_1.cancelCombinator(saveCancelFn => refreshStrategy.bind(this)(client, newVersionEntry, saveCancelFn, invalidationKeys));
     }
     logExecutingSql(payload) {
         this.logger('Executing Load Pre Aggregation SQL', payload);
@@ -502,9 +512,10 @@ class PreAggregationLoader {
             targetTableName,
             requestId: this.requestId,
             newVersionEntry,
+            buildRangeEnd: this.preAggregation.buildRangeEnd,
         };
     }
-    async refreshImplStoreInSourceStrategy(client, newVersionEntry, saveCancelFn, preAggregation, invalidationKeys) {
+    async refreshImplStoreInSourceStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
         const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
         const targetTableName = this.targetTableName(newVersionEntry);
         const query = QueryCache_1.QueryCache.replacePreAggregationTableNames(loadSql, this.preAggregationsTablesToTempTables)
@@ -526,7 +537,7 @@ class PreAggregationLoader {
     /**
      * Strategy to copy pre-aggregation from source db (with write permissions) to external data
      */
-    async refreshImplTempTableExternalStrategy(client, newVersionEntry, saveCancelFn, preAggregation, invalidationKeys) {
+    async refreshImplTempTableExternalStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
         const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
         await client.createSchemaIfNotExists(this.preAggregation.preAggregationsSchema);
         const targetTableName = this.targetTableName(newVersionEntry);
@@ -536,7 +547,7 @@ class PreAggregationLoader {
         this.logExecutingSql(queryOptions);
         await saveCancelFn(client.loadPreAggregationIntoTable(targetTableName, query, params, queryOptions));
         try {
-            const tableData = await this.downloadTempExternalPreAggregation(client, newVersionEntry, preAggregation, saveCancelFn, queryOptions);
+            const tableData = await this.downloadTempExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions);
             try {
                 await this.uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions);
             }
@@ -555,7 +566,7 @@ class PreAggregationLoader {
     /**
      * Strategy to copy pre-aggregation from source db (for read-only permissions) to external data
      */
-    async refreshImplStreamExternalStrategy(client, newVersionEntry, saveCancelFn, preAggregation, invalidationKeys) {
+    async refreshImplStreamExternalStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
         const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
         // @todo Deprecated, BaseDriver already implements it, before remove we need to add check for factoryDriver
         if (!client.downloadQueryResults) {
@@ -600,7 +611,7 @@ class PreAggregationLoader {
     /**
      * Create table (for db with write permissions) and extract data via memory/stream/unload
      */
-    async downloadTempExternalPreAggregation(client, newVersionEntry, preAggregation, saveCancelFn, queryOptions) {
+    async downloadTempExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions) {
         // @todo Deprecated, BaseDriver already implements it, before remove we need to add check for factoryDriver
         if (!client.downloadTable) {
             throw new Error('Can\'t load external pre-aggregation: source driver doesn\'t support downloadTable()');
@@ -617,7 +628,7 @@ class PreAggregationLoader {
             else if (capabilities.streamImport && client.stream) {
                 tableData = await saveCancelFn(client.stream(`SELECT * FROM ${table}`, [], this.getStreamingOptions()));
                 if (client.unload) {
-                    const stream = new StreamObjectsCounter_1.LargeStreamWarning(preAggregation.preAggregationId, (msg) => {
+                    const stream = new StreamObjectsCounter_1.LargeStreamWarning(this.preAggregation.preAggregationId, (msg) => {
                         this.logger('Downloading external pre-aggregation warning', {
                             ...queryOptions,
                             error: msg
@@ -711,7 +722,10 @@ exports.PreAggregationLoader = PreAggregationLoader;
 class PreAggregationPartitionRangeLoader {
     constructor(redisPrefix, driverFactory, logger, queryCache, 
     // eslint-disable-next-line no-use-before-define
-    preAggregations, preAggregation, preAggregationsTablesToTempTables, loadCache, options = {}) {
+    preAggregations, preAggregation, preAggregationsTablesToTempTables, loadCache, options = {
+        maxPartitions: 10000,
+        maxSourceRowLimit: 10000,
+    }) {
         this.redisPrefix = redisPrefix;
         this.driverFactory = driverFactory;
         this.logger = logger;
@@ -723,11 +737,16 @@ class PreAggregationPartitionRangeLoader {
         this.options = options;
         this.waitForRenew = options.waitForRenew;
         this.requestId = options.requestId;
+        this.lambdaQuery = options.lambdaQuery;
         this.dataSource = preAggregation.dataSource;
     }
     async loadRangeQuery(rangeQuery, partitionRange) {
         const [query, values, queryOptions] = rangeQuery;
-        return this.queryCache.cacheQueryResult(query, values, QueryCache_1.QueryCache.queryCacheKey({ query, values }), 24 * 60 * 60, {
+        const invalidate = this.preAggregation.invalidateKeyQueries &&
+            this.preAggregation.invalidateKeyQueries[0]
+            ? this.preAggregation.invalidateKeyQueries[0].slice(0, 2)
+            : false;
+        return this.queryCache.cacheQueryResult(query, values, QueryCache_1.QueryCache.queryCacheKey({ query, values, invalidate }), 24 * 60 * 60, {
             renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold
                 || (queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.renewalThreshold) || 24 * 60 * 60,
             waitForRenew: this.waitForRenew,
@@ -736,7 +755,7 @@ class PreAggregationPartitionRangeLoader {
             dataSource: this.dataSource,
             useInMemory: true,
             external: queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.external,
-            renewalKey: partitionRange ? await this.getInvalidationKeyValues(partitionRange) : null
+            renewalKey: partitionRange ? await this.getInvalidationKeyValues(partitionRange) : null,
         });
     }
     getInvalidationKeyValues(range) {
@@ -779,32 +798,38 @@ class PreAggregationPartitionRangeLoader {
             }), {
                 ...options,
                 renewalThreshold: (options === null || options === void 0 ? void 0 : options.incremental) && updateWindowToBoundary < new Date() ?
-                    // if updateWindowToBoundary passed just moments ago we want to renew it earlier in case of server
-                    // and db clock don't match
+                    // if updateWindowToBoundary passed just moments ago we want to renew it earlier in case
+                    // of server and db clock don't match
                     Math.min(Math.round((new Date().getTime() - updateWindowToBoundary.getTime()) / 1000), options === null || options === void 0 ? void 0 : options.renewalThresholdOutsideUpdateWindow) : options === null || options === void 0 ? void 0 : options.renewalThreshold
             }];
     }
-    partitionPreAggregationDescription(range) {
+    partitionPreAggregationDescription(range, buildRange) {
         const partitionTableName = PreAggregationPartitionRangeLoader.partitionTableName(this.preAggregation.tableName, this.preAggregation.partitionGranularity, range);
+        const [_, buildRangeEnd] = buildRange;
+        const loadRange = [...range];
+        if (this.preAggregation.unionWithSourceData && buildRangeEnd < range[1]) {
+            loadRange[1] = buildRangeEnd;
+        }
         return {
             ...this.preAggregation,
             tableName: partitionTableName,
             loadSql: this.preAggregation.loadSql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.loadSql, range, partitionTableName),
+                this.replacePartitionSqlAndParams(this.preAggregation.loadSql, loadRange, partitionTableName),
             sql: this.preAggregation.sql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.sql, range, partitionTableName),
+                this.replacePartitionSqlAndParams(this.preAggregation.sql, loadRange, partitionTableName),
             invalidateKeyQueries: (this.preAggregation.invalidateKeyQueries || [])
                 .map(q => this.replacePartitionSqlAndParams(q, range, partitionTableName)),
             indexesSql: (this.preAggregation.indexesSql || [])
                 .map(q => ({ ...q, sql: this.replacePartitionSqlAndParams(q.sql, range, partitionTableName) })),
             previewSql: this.preAggregation.previewSql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.previewSql, range, partitionTableName)
+                this.replacePartitionSqlAndParams(this.preAggregation.previewSql, range, partitionTableName),
+            buildRangeEnd,
         };
     }
     async loadPreAggregations() {
         if (this.preAggregation.partitionGranularity && !this.preAggregation.expandedPartition) {
-            const partitionRanges = await this.partitionRanges();
-            const partitionLoaders = partitionRanges.map(range => new PreAggregationLoader(this.redisPrefix, this.driverFactory, this.logger, this.queryCache, this.preAggregations, this.partitionPreAggregationDescription(range), this.preAggregationsTablesToTempTables, this.loadCache, this.options));
+            const { buildRange, partitionRanges } = await this.partitionRanges();
+            const partitionLoaders = partitionRanges.map(range => new PreAggregationLoader(this.redisPrefix, this.driverFactory, this.logger, this.queryCache, this.preAggregations, this.partitionPreAggregationDescription(range, buildRange), this.preAggregationsTablesToTempTables, this.loadCache, this.options));
             const resolveResults = await Promise.all(partitionLoaders.map(l => l.loadPreAggregation(false)));
             const loadResults = resolveResults.filter(res => res !== null);
             if (this.options.externalRefresh && loadResults.length === 0) {
@@ -813,25 +838,65 @@ class PreAggregationPartitionRangeLoader {
                     'built yet. Please make sure your refresh worker is configured ' +
                     'correctly and running.');
             }
-            const allTableTargetNames = loadResults
-                .map(targetTableName => targetTableName.targetTableName);
+            const allTableTargetNames = loadResults.map(targetTableName => targetTableName.targetTableName);
+            let lastUpdatedAt = getLastUpdatedAtTimestamp(loadResults.map(r => r.lastUpdatedAt));
+            let lambdaTable;
+            if (this.lambdaQuery && loadResults.length > 0) {
+                const { buildRangeEnd } = loadResults[loadResults.length - 1];
+                lambdaTable = await this.downloadLambdaTable(buildRangeEnd);
+                allTableTargetNames.push(lambdaTable.name);
+                lastUpdatedAt = Date.now();
+            }
             const unionTargetTableName = allTableTargetNames
                 .map(targetTableName => `SELECT * FROM ${targetTableName}`)
                 .join(' UNION ALL ');
             return {
                 targetTableName: allTableTargetNames.length === 1 ? allTableTargetNames[0] : `(${unionTargetTableName})`,
                 refreshKeyValues: loadResults.map(t => t.refreshKeyValues),
-                lastUpdatedAt: getLastUpdatedAtTimestamp(loadResults.map(r => r.lastUpdatedAt)),
+                lastUpdatedAt,
+                buildRangeEnd: undefined,
+                lambdaTable,
             };
         }
         else {
             return new PreAggregationLoader(this.redisPrefix, this.driverFactory, this.logger, this.queryCache, this.preAggregations, this.preAggregation, this.preAggregationsTablesToTempTables, this.loadCache, this.options).loadPreAggregation(true);
         }
     }
+    /**
+     * Downloads the lambda table from the source DB.
+     */
+    async downloadLambdaTable(fromDate) {
+        const { sqlAndParams, cacheKeyQueries } = this.lambdaQuery;
+        const [query, params] = sqlAndParams;
+        const values = params.map((p) => {
+            if (p === shared_1.FROM_PARTITION_RANGE) {
+                return fromDate;
+            }
+            if (p === shared_1.MAX_SOURCE_ROW_LIMIT) {
+                return this.options.maxSourceRowLimit;
+            }
+            return p;
+        });
+        const { data } = await this.queryCache.renewQuery(query, values, cacheKeyQueries, 60 * 60, [query, values], undefined, {
+            requestId: this.requestId,
+            skipRefreshKeyWaitForRenew: false,
+            dataSource: this.dataSource,
+            external: false,
+            useCsvQuery: true,
+        });
+        if (data.rowCount === this.options.maxSourceRowLimit) {
+            throw new Error(`The maximum number of source rows ${this.options.maxSourceRowLimit} was reached for ${this.preAggregation.preAggregationId}`);
+        }
+        return {
+            name: `${exports.LAMBDA_TABLE_PREFIX}_${this.preAggregation.tableName.replace('.', '_')}`,
+            columns: data.types,
+            csvRows: data.csvRows,
+        };
+    }
     async partitionPreAggregations() {
         if (this.preAggregation.partitionGranularity && !this.preAggregation.expandedPartition) {
-            const partitionRanges = await this.partitionRanges();
-            return partitionRanges.map(range => this.partitionPreAggregationDescription(range));
+            const { buildRange, partitionRanges } = await this.partitionRanges();
+            return partitionRanges.map(range => this.partitionPreAggregationDescription(range, buildRange));
         }
         else {
             return [this.preAggregation];
@@ -840,7 +905,7 @@ class PreAggregationPartitionRangeLoader {
     async partitionRanges() {
         const buildRange = await this.loadBuildRange();
         if (!buildRange[0] || !buildRange[1]) {
-            return [];
+            return { buildRange, partitionRanges: [] };
         }
         let dateRange = PreAggregationPartitionRangeLoader.intersectDateRanges(buildRange, this.preAggregation.matchedTimeDimensionDateRange);
         if (!dateRange) {
@@ -848,7 +913,11 @@ class PreAggregationPartitionRangeLoader {
             // use last partition so outer query can receive expected table structure.
             dateRange = [buildRange[1], buildRange[1]];
         }
-        return PreAggregationPartitionRangeLoader.timeSeries(this.preAggregation.partitionGranularity, dateRange);
+        const partitionRanges = PreAggregationPartitionRangeLoader.timeSeries(this.preAggregation.partitionGranularity, dateRange);
+        if (partitionRanges.length > this.options.maxPartitions) {
+            throw new Error(`The maximum number of partitions (${this.options.maxPartitions}) was reached for the pre-aggregation`);
+        }
+        return { buildRange: dateRange, partitionRanges };
     }
     async loadBuildRange() {
         const { preAggregationStartEndQueries } = this.preAggregation;
@@ -860,9 +929,12 @@ class PreAggregationPartitionRangeLoader {
         const [rangeStart, rangeEnd] = await Promise.all(preAggregationStartEndQueries.map(async (rangeQuery, i) => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1]))));
         return this.orNowIfEmpty([rangeStart, rangeEnd]);
     }
+    now() {
+        return shared_1.utcToLocalTimeZone(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
+    }
     orNowIfEmpty(dateRange) {
         if (!dateRange[0] && !dateRange[1]) {
-            const now = shared_1.utcToLocalTimeZone(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
+            const now = this.now();
             return [now, now];
         }
         if (!dateRange[0]) {
@@ -973,16 +1045,21 @@ class PreAggregations {
         };
         let queryParamsReplacement = null;
         const preAggregationsTablesToTempTablesPromise = preAggregations.map((p, i) => (preAggregationsTablesToTempTables) => {
+            var _a;
             const loader = new PreAggregationPartitionRangeLoader(this.redisPrefix, () => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, preAggregationsTablesToTempTables, getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
+                maxPartitions: this.options.maxPartitions,
+                maxSourceRowLimit: this.options.maxSourceRowLimit,
                 waitForRenew: queryBody.renewQuery,
                 // TODO workaround to avoid continuous waiting on building pre-aggregation dependencies
                 forceBuild: i === preAggregations.length - 1 ? queryBody.forceBuildPreAggregations : false,
                 requestId: queryBody.requestId,
                 metadata: queryBody.metadata,
                 orphanedTimeout: queryBody.orphanedTimeout,
+                lambdaQuery: ((_a = queryBody.lambdaQueries) !== null && _a !== void 0 ? _a : {})[p.preAggregationId],
                 externalRefresh: this.externalRefresh
             });
-            const preAggregationPromise = () => loader.loadPreAggregations().then(async (loadResult) => {
+            const preAggregationPromise = async () => {
+                const loadResult = await loader.loadPreAggregations();
                 const usedPreAggregation = {
                     ...loadResult,
                     type: p.type,
@@ -992,7 +1069,7 @@ class PreAggregations {
                     queryParamsReplacement = await loader.replaceQueryBuildRangeParams(queryBody.values);
                 }
                 return [p.tableName, usedPreAggregation];
-            });
+            };
             return preAggregationPromise().then(res => preAggregationsTablesToTempTables.concat([res]));
         }).reduce((promise, fn) => promise.then(fn), Promise.resolve([]));
         return preAggregationsTablesToTempTablesPromise.then(preAggregationsTablesToTempTables => ({
@@ -1027,9 +1104,11 @@ class PreAggregations {
         };
         const expandedPreAggregations = await Promise.all(preAggregations.map(p => {
             const loader = new PreAggregationPartitionRangeLoader(this.redisPrefix, () => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, [], getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
+                maxPartitions: this.options.maxPartitions,
+                maxSourceRowLimit: this.options.maxSourceRowLimit,
                 waitForRenew: queryBody.renewQuery,
                 requestId: queryBody.requestId,
-                externalRefresh: this.externalRefresh
+                externalRefresh: this.externalRefresh,
             });
             return loader.partitionPreAggregations();
         }));
@@ -1045,9 +1124,9 @@ class PreAggregations {
     async getQueue(dataSource = 'default') {
         if (!this.queue[dataSource]) {
             this.queue[dataSource] = QueryCache_1.QueryCache.createQueue(`SQL_PRE_AGGREGATIONS_${this.redisPrefix}_${dataSource}`, () => this.driverFactory(dataSource), (client, q) => {
-                const { preAggregation, preAggregationsTablesToTempTables, newVersionEntry, requestId, invalidationKeys } = q;
-                const loader = new PreAggregationLoader(this.redisPrefix, () => this.driverFactory(dataSource), this.logger, this.queryCache, this, preAggregation, preAggregationsTablesToTempTables, new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, { requestId, dataSource }), { requestId, externalRefresh: this.externalRefresh });
-                return loader.refresh(preAggregation, newVersionEntry, invalidationKeys)(client);
+                const { preAggregation, preAggregationsTablesToTempTables, newVersionEntry, requestId, invalidationKeys, buildRangeEnd } = q;
+                const loader = new PreAggregationLoader(this.redisPrefix, () => this.driverFactory(dataSource), this.logger, this.queryCache, this, preAggregation, preAggregationsTablesToTempTables, new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, { requestId, dataSource }), { requestId, externalRefresh: this.externalRefresh, buildRangeEnd });
+                return loader.refresh(newVersionEntry, invalidationKeys, client);
             }, {
                 concurrency: 1,
                 logger: this.logger,