@cubejs-backend/query-orchestrator 1.2.13 → 1.2.14

package/dist/src/orchestrator/PreAggregations.js

@@ -3,15 +3,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PreAggregations = exports.PreAggregationPartitionRangeLoader = exports.PreAggregationLoader = exports.getLastUpdatedAtTimestamp = exports.LAMBDA_TABLE_PREFIX = void 0;
-const crypto_1 = __importDefault(require("crypto"));
+exports.PreAggregations = exports.tablesToVersionEntries = exports.getStructureVersion = exports.getLastUpdatedAtTimestamp = exports.version = exports.LAMBDA_TABLE_PREFIX = void 0;
 const ramda_1 = __importDefault(require("ramda"));
+const crypto_1 = __importDefault(require("crypto"));
 const shared_1 = require("@cubejs-backend/shared");
-const base_driver_1 = require("@cubejs-backend/base-driver");
 const lru_cache_1 = __importDefault(require("lru-cache"));
 const QueryCache_1 = require("./QueryCache");
-const ContinueWaitError_1 = require("./ContinueWaitError");
-const StreamObjectsCounter_1 = require("./StreamObjectsCounter");
+const PreAggregationPartitionRangeLoader_1 = require("./PreAggregationPartitionRangeLoader");
+const PreAggregationLoader_1 = require("./PreAggregationLoader");
+const PreAggregationLoadCache_1 = require("./PreAggregationLoadCache");
 /// Name of the inline table containing the lambda rows.
 exports.LAMBDA_TABLE_PREFIX = 'lambda';
 function encodeTimeStamp(time) {
@@ -42,11 +42,7 @@ function version(cacheKey) {
     result += hashCharset.charAt(residue % 32);
     return result;
 }
-// There’re community developed and custom drivers which not always up-to-date with latest BaseDriver.
-// Extra defence for drivers that don't expose now() yet.
-function nowTimestamp(client) {
-    return client.nowTimestamp?.() ?? new Date().getTime();
-}
+exports.version = version;
 // Returns the oldest timestamp, if any.
 function getLastUpdatedAtTimestamp(timestamps) {
     timestamps = timestamps.filter(t => t !== undefined);
@@ -71,6 +67,7 @@ function getStructureVersion(preAggregation) {
     }
     return version(versionArray.length === 1 ? versionArray[0] : versionArray);
 }
+exports.getStructureVersion = getStructureVersion;
 const tablesToVersionEntries = (schema, tables) => ramda_1.default.sortBy(table => -table.last_updated_at, tables.map(table => {
     const match = (table.table_name || table.TABLE_NAME).match(/(.+)_(.+)_(.+)_(.+)/);
     if (!match) {
@@ -93,1247 +90,7 @@ const tablesToVersionEntries = (schema, tables) => ramda_1.default.sortBy(table
     }
     return entity;
 }).filter(ramda_1.default.identity));
-class PreAggregationLoadCache {
-    redisPrefix;
-    driverFactory;
-    queryCache;
-    // eslint-disable-next-line no-use-before-define
-    preAggregations;
-    queryResults;
-    externalDriverFactory;
-    requestId;
-    versionEntries;
-    tables;
-    tableColumnTypes;
-    // TODO this is in memory cache structure as well however it depends on
-    // data source only and load cache is per data source for now.
-    // Make it per data source key in case load cache scope is broaden.
-    queryStageState;
-    dataSource;
-    tablePrefixes;
-    constructor(redisPrefix, clientFactory, queryCache, preAggregations, options = { dataSource: 'default' }) {
-        this.redisPrefix = `${redisPrefix}_${options.dataSource}`;
-        this.dataSource = options.dataSource;
-        this.driverFactory = clientFactory;
-        this.queryCache = queryCache;
-        this.preAggregations = preAggregations;
-        this.queryResults = {};
-        this.externalDriverFactory = preAggregations.externalDriverFactory;
-        this.requestId = options.requestId;
-        this.tablePrefixes = options.tablePrefixes;
-        this.versionEntries = {};
-        this.tables = {};
-        this.tableColumnTypes = {};
-    }
-    async tablesFromCache(preAggregation, forceRenew) {
-        let tables = forceRenew ? null : await this.queryCache.getCacheDriver().get(this.tablesCachePrefixKey(preAggregation));
-        if (!tables) {
-            tables = await this.preAggregations.getLoadCacheQueue(this.dataSource).executeInQueue('query', `Fetch tables for ${preAggregation.preAggregationsSchema}`, {
-                preAggregation, requestId: this.requestId
-            }, 0, { requestId: this.requestId });
-        }
-        return tables;
-    }
-    async fetchTables(preAggregation) {
-        if (preAggregation.external && !this.externalDriverFactory) {
-            throw new Error('externalDriverFactory is not provided. Please use CUBEJS_DEV_MODE=true or provide Cube Store connection env variables for production usage.');
-        }
-        const newTables = await this.fetchTablesNoCache(preAggregation);
-        await this.queryCache.getCacheDriver().set(this.tablesCachePrefixKey(preAggregation), newTables, this.preAggregations.options.preAggregationsSchemaCacheExpire || 60 * 60);
-        return newTables;
-    }
-    async fetchTablesNoCache(preAggregation) {
-        const client = preAggregation.external ?
-            await this.externalDriverFactory() :
-            await this.driverFactory();
-        if (this.tablePrefixes && client.getPrefixTablesQuery && this.preAggregations.options.skipExternalCacheAndQueue) {
-            return client.getPrefixTablesQuery(preAggregation.preAggregationsSchema, this.tablePrefixes);
-        }
-        return client.getTablesQuery(preAggregation.preAggregationsSchema);
-    }
-    tablesCachePrefixKey(preAggregation) {
-        return this.queryCache.getKey('SQL_PRE_AGGREGATIONS_TABLES', `${preAggregation.dataSource}${preAggregation.preAggregationsSchema}${preAggregation.external ? '_EXT' : ''}`);
-    }
-    async getTablesQuery(preAggregation) {
-        const redisKey = this.tablesCachePrefixKey(preAggregation);
-        if (!this.tables[redisKey]) {
-            const tables = this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external ?
-                await this.fetchTablesNoCache(preAggregation) :
-                await this.tablesFromCache(preAggregation);
-            if (tables === undefined) {
-                throw new Error('Pre-aggregation tables are undefined.');
-            }
-            this.tables[redisKey] = tables;
-        }
-        return this.tables[redisKey];
-    }
-    async getTableColumnTypes(preAggregation, tableName) {
-        const prefixKey = this.tablesCachePrefixKey(preAggregation);
-        if (!this.tableColumnTypes[prefixKey]?.[tableName]) {
-            if (!this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external) {
-                throw new Error(`Lambda union with source data feature is supported only by external rollups stored in Cube Store but was invoked for '${preAggregation.preAggregationId}'`);
-            }
-            const client = await this.externalDriverFactory();
-            const columnTypes = await client.tableColumnTypes(tableName);
-            if (!this.tableColumnTypes[prefixKey]) {
-                this.tableColumnTypes[prefixKey] = {};
-            }
-            this.tableColumnTypes[prefixKey][tableName] = columnTypes;
-        }
-        return this.tableColumnTypes[prefixKey][tableName];
-    }
-    async calculateVersionEntries(preAggregation) {
-        let versionEntries = tablesToVersionEntries(preAggregation.preAggregationsSchema, await this.getTablesQuery(preAggregation));
-        // It presumes strong consistency guarantees for external pre-aggregation tables ingestion
-        if (!preAggregation.external) {
-            // eslint-disable-next-line
-            const [active, toProcess, queries] = await this.fetchQueryStageState();
-            const targetTableNamesInQueue = (Object.keys(queries))
-                // eslint-disable-next-line no-use-before-define
-                .map(q => PreAggregations.targetTableName(queries[q].query.newVersionEntry));
-            versionEntries = versionEntries.filter(
-            // eslint-disable-next-line no-use-before-define
-            e => targetTableNamesInQueue.indexOf(PreAggregations.targetTableName(e)) === -1);
-        }
-        const byContent = {};
-        const byStructure = {};
-        const byTableName = {};
-        versionEntries.forEach(e => {
-            const contentKey = `${e.table_name}_${e.content_version}`;
-            if (!byContent[contentKey]) {
-                byContent[contentKey] = e;
-            }
-            const structureKey = `${e.table_name}_${e.structure_version}`;
-            if (!byStructure[structureKey]) {
-                byStructure[structureKey] = e;
-            }
-            if (!byTableName[e.table_name]) {
-                byTableName[e.table_name] = e;
-            }
-        });
-        return { versionEntries, byContent, byStructure, byTableName };
-    }
-    async getVersionEntries(preAggregation) {
-        if (this.tablePrefixes && !this.tablePrefixes.find(p => preAggregation.tableName.split('.')[1].startsWith(p))) {
-            throw new Error(`Load cache tries to load table ${preAggregation.tableName} outside of tablePrefixes filter: ${this.tablePrefixes.join(', ')}`);
-        }
-        const redisKey = this.tablesCachePrefixKey(preAggregation);
-        if (!this.versionEntries[redisKey]) {
-            this.versionEntries[redisKey] = this.calculateVersionEntries(preAggregation).catch(e => {
-                delete this.versionEntries[redisKey];
-                throw e;
-            });
-        }
-        return this.versionEntries[redisKey];
-    }
-    async keyQueryResult(sqlQuery, waitForRenew, priority) {
-        const [query, values, queryOptions] = Array.isArray(sqlQuery) ? sqlQuery : [sqlQuery, [], {}];
-        if (!this.queryResults[this.queryCache.queryRedisKey([query, values])]) {
-            this.queryResults[this.queryCache.queryRedisKey([query, values])] = await this.queryCache.cacheQueryResult(query, values, [query, values], 60 * 60, {
-                renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold
-                    || queryOptions?.renewalThreshold || 2 * 60,
-                renewalKey: [query, values],
-                waitForRenew,
-                priority,
-                requestId: this.requestId,
-                dataSource: this.dataSource,
-                useInMemory: true,
-                external: queryOptions?.external
-            });
-        }
-        return this.queryResults[this.queryCache.queryRedisKey([query, values])];
-    }
-    hasKeyQueryResult(keyQuery) {
-        return !!this.queryResults[this.queryCache.queryRedisKey(keyQuery)];
-    }
-    async getQueryStage(stageQueryKey) {
-        const queue = await this.preAggregations.getQueue(this.dataSource);
-        await this.fetchQueryStageState(queue);
-        return queue.getQueryStage(stageQueryKey, undefined, this.queryStageState);
-    }
-    async fetchQueryStageState(queue) {
-        queue = queue || await this.preAggregations.getQueue(this.dataSource);
-        if (!this.queryStageState) {
-            this.queryStageState = await queue.fetchQueryStageState();
-        }
-        return this.queryStageState;
-    }
-    async reset(preAggregation) {
-        await this.tablesFromCache(preAggregation, true);
-        this.tables = {};
-        this.tableColumnTypes = {};
-        this.queryStageState = undefined;
-        this.versionEntries = {};
-    }
-}
-class PreAggregationLoader {
-    redisPrefix;
-    driverFactory;
-    logger;
-    queryCache;
-    loadCache;
-    // eslint-disable-next-line no-use-before-define
-    preAggregations;
-    preAggregation;
-    preAggregationsTablesToTempTables;
-    /**
-     * Determines whether current instance instantiated for a jobed build query
-     * (initialized by the /cubejs-system/v1/pre-aggregations/jobs endpoint) or
-     * not.
-     */
-    isJob;
-    waitForRenew;
-    forceBuild;
-    orphanedTimeout;
-    externalDriverFactory;
-    requestId;
-    metadata;
-    structureVersionPersistTime;
-    externalRefresh;
-    constructor(redisPrefix, driverFactory, logger, queryCache, 
-    // eslint-disable-next-line no-use-before-define
-    preAggregations, preAggregation, preAggregationsTablesToTempTables, loadCache, options = {}) {
-        this.redisPrefix = redisPrefix;
-        this.driverFactory = driverFactory;
-        this.logger = logger;
-        this.queryCache = queryCache;
-        this.loadCache = loadCache;
-        this.preAggregations = preAggregations;
-        this.preAggregation = preAggregation;
-        this.preAggregationsTablesToTempTables = preAggregationsTablesToTempTables;
-        this.isJob = !!options.isJob;
-        this.waitForRenew = options.waitForRenew;
-        this.forceBuild = options.forceBuild;
-        this.orphanedTimeout = options.orphanedTimeout;
-        this.externalDriverFactory = preAggregations.externalDriverFactory;
-        this.requestId = options.requestId;
-        this.metadata = options.metadata;
-        this.structureVersionPersistTime = preAggregations.structureVersionPersistTime;
-        this.externalRefresh = options.externalRefresh;
-        if (this.externalRefresh && this.waitForRenew) {
-            const message = 'Invalid configuration - when externalRefresh is true, it will not perform a renew, therefore you cannot wait for it using waitForRenew.';
-            if (['production', 'test'].includes((0, shared_1.getEnv)('nodeEnv'))) {
-                throw new Error(message);
-            }
-            else {
-                this.logger('Invalid Configuration', {
-                    requestId: this.requestId,
-                    warning: message,
-                });
-                this.waitForRenew = false;
-            }
-        }
-    }
-    async loadPreAggregation(throwOnMissingPartition) {
-        const notLoadedKey = (this.preAggregation.invalidateKeyQueries || [])
-            .find(keyQuery => !this.loadCache.hasKeyQueryResult(keyQuery));
-        if (this.isJob || !(notLoadedKey && !this.waitForRenew)) {
-            // Case 1: pre-agg build job processing.
-            // Case 2: either we have no data cached for this rollup or waitForRenew
-            // is true, either way, synchronously renew what data is needed so that
-            // the most current data will be returned fo the current request.
-            const result = await this.loadPreAggregationWithKeys();
-            const refreshKeyValues = await this.getInvalidationKeyValues();
-            return {
-                ...result,
-                refreshKeyValues,
-                queryKey: this.isJob
-                    // We need to return a queryKey value for the jobed build query
-                    // (initialized by the /cubejs-system/v1/pre-aggregations/jobs
-                    // endpoint) as a part of the response to make it possible to get a
-                    // query result from the cache by the other API call.
-                    ? this.preAggregationQueryKey(refreshKeyValues)
-                    : undefined,
-            };
-        }
-        else {
-            // Case 3: pre-agg is exists
-            const structureVersion = getStructureVersion(this.preAggregation);
-            const getVersionsStarted = new Date();
-            const { byStructure } = await this.loadCache.getVersionEntries(this.preAggregation);
-            this.logger('Load PreAggregations Tables', {
-                preAggregation: this.preAggregation,
-                requestId: this.requestId,
-                duration: (new Date().getTime() - getVersionsStarted.getTime())
-            });
-            const versionEntryByStructureVersion = byStructure[`${this.preAggregation.tableName}_${structureVersion}`];
-            if (this.externalRefresh) {
-                if (!versionEntryByStructureVersion && throwOnMissingPartition) {
-                    // eslint-disable-next-line no-use-before-define
-                    throw new Error(PreAggregations.noPreAggregationPartitionsBuiltMessage([this.preAggregation]));
-                }
-                if (!versionEntryByStructureVersion) {
-                    return null;
-                }
-                else {
-                    // the rollups are being maintained independently of this instance of cube.js
-                    // immediately return the latest rollup data that instance already has
-                    return {
-                        targetTableName: this.targetTableName(versionEntryByStructureVersion),
-                        refreshKeyValues: [],
-                        lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
-                        buildRangeEnd: versionEntryByStructureVersion.build_range_end,
-                    };
-                }
-            }
-            if (versionEntryByStructureVersion) {
-                // this triggers an asyncronous/background load of the pre-aggregation but immediately
-                // returns the latest data it already has
-                this.loadPreAggregationWithKeys().catch(e => {
-                    if (!(e instanceof ContinueWaitError_1.ContinueWaitError)) {
-                        this.logger('Error loading pre-aggregation', {
-                            error: (e.stack || e),
-                            preAggregation: this.preAggregation,
-                            requestId: this.requestId
-                        });
-                    }
-                });
-                return {
-                    targetTableName: this.targetTableName(versionEntryByStructureVersion),
-                    refreshKeyValues: [],
-                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
-                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
-                };
-            }
-            else {
-                // no rollup has been built yet - build it synchronously as part of responding to this request
-                return this.loadPreAggregationWithKeys();
-            }
-        }
-    }
-    async loadPreAggregationWithKeys() {
-        const invalidationKeys = await this.getPartitionInvalidationKeyValues();
-        const contentVersion = this.contentVersion(invalidationKeys);
-        const structureVersion = getStructureVersion(this.preAggregation);
-        const versionEntries = await this.loadCache.getVersionEntries(this.preAggregation);
-        const getVersionEntryByContentVersion = ({ byContent }) => byContent[`${this.preAggregation.tableName}_${contentVersion}`];
-        const versionEntryByContentVersion = getVersionEntryByContentVersion(versionEntries);
-        if (versionEntryByContentVersion && !this.forceBuild) {
-            const targetTableName = this.targetTableName(versionEntryByContentVersion);
-            // No need to block here
-            this.updateLastTouch(targetTableName);
-            return {
-                targetTableName,
-                refreshKeyValues: [],
-                lastUpdatedAt: versionEntryByContentVersion.last_updated_at,
-                buildRangeEnd: versionEntryByContentVersion.build_range_end,
-            };
-        }
-        if (!this.waitForRenew && !this.forceBuild) {
-            const versionEntryByStructureVersion = versionEntries.byStructure[`${this.preAggregation.tableName}_${structureVersion}`];
-            if (versionEntryByStructureVersion) {
-                const targetTableName = this.targetTableName(versionEntryByStructureVersion);
-                // No need to block here
-                this.updateLastTouch(targetTableName);
-                return {
-                    targetTableName,
-                    refreshKeyValues: [],
-                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
-                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
-                };
-            }
-        }
-        const client = this.preAggregation.external ?
-            await this.externalDriverFactory() :
-            await this.driverFactory();
-        if (!versionEntries.versionEntries.length) {
-            await client.createSchemaIfNotExists(this.preAggregation.preAggregationsSchema);
-        }
-        // ensure we find appropriate structure version before invalidating anything
-        const versionEntry = versionEntries.byStructure[`${this.preAggregation.tableName}_${structureVersion}`] ||
-            versionEntries.byTableName[this.preAggregation.tableName];
-        const newVersionEntry = {
-            table_name: this.preAggregation.tableName,
-            structure_version: structureVersion,
-            content_version: contentVersion,
-            last_updated_at: nowTimestamp(client),
-            naming_version: 2,
-        };
-        const mostRecentResult = async () => {
-            await this.loadCache.reset(this.preAggregation);
-            const lastVersion = getVersionEntryByContentVersion(await this.loadCache.getVersionEntries(this.preAggregation));
-            if (!lastVersion) {
-                throw new Error(`Pre-aggregation table is not found for ${this.preAggregation.tableName} after it was successfully created`);
-            }
-            const targetTableName = this.targetTableName(lastVersion);
-            this.updateLastTouch(targetTableName);
-            return {
-                targetTableName,
-                refreshKeyValues: [],
-                lastUpdatedAt: lastVersion.last_updated_at,
-                buildRangeEnd: lastVersion.build_range_end,
-            };
-        };
-        if (this.forceBuild) {
-            this.logger('Force build pre-aggregation', {
-                preAggregation: this.preAggregation,
-                requestId: this.requestId,
-                metadata: this.metadata,
-                queryKey: this.preAggregationQueryKey(invalidationKeys),
-                newVersionEntry
-            });
-            if (this.isJob) {
-                // We don't want to wait for the jobed build query result. So we run the
-                // executeInQueue method and immediately return the LoadPreAggregationResult object.
-                this
-                    .executeInQueue(invalidationKeys, this.priority(10), newVersionEntry)
-                    .catch((e) => {
-                    this.logger('Pre-aggregations build job error', {
-                        preAggregation: this.preAggregation,
-                        requestId: this.requestId,
-                        newVersionEntry,
-                        error: (e.stack || e),
-                    });
-                });
-                const targetTableName = this.targetTableName(newVersionEntry);
-                this.updateLastTouch(targetTableName);
-                return {
-                    targetTableName,
-                    refreshKeyValues: [],
-                    lastUpdatedAt: newVersionEntry.last_updated_at,
-                    buildRangeEnd: this.preAggregation.buildRangeEnd,
-                };
-            }
-            else {
-                await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
-                return mostRecentResult();
-            }
-        }
-        if (versionEntry) {
-            if (versionEntry.structure_version !== newVersionEntry.structure_version) {
-                this.logger('Invalidating pre-aggregation structure', {
-                    preAggregation: this.preAggregation,
-                    requestId: this.requestId,
-                    queryKey: this.preAggregationQueryKey(invalidationKeys),
-                    newVersionEntry
-                });
-                await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
-                return mostRecentResult();
-            }
-            else if (versionEntry.content_version !== newVersionEntry.content_version) {
-                if (this.waitForRenew) {
-                    this.logger('Waiting for pre-aggregation renew', {
-                        preAggregation: this.preAggregation,
-                        requestId: this.requestId,
-                        queryKey: this.preAggregationQueryKey(invalidationKeys),
-                        newVersionEntry
-                    });
-                    await this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry);
-                    return mostRecentResult();
-                }
-                else {
-                    this.scheduleRefresh(invalidationKeys, newVersionEntry);
-                }
-            }
-        }
-        else {
-            this.logger('Creating pre-aggregation from scratch', {
-                preAggregation: this.preAggregation,
-                requestId: this.requestId,
-                queryKey: this.preAggregationQueryKey(invalidationKeys),
-                newVersionEntry
-            });
-            await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
-            return mostRecentResult();
-        }
-        const targetTableName = this.targetTableName(versionEntry);
-        this.updateLastTouch(targetTableName);
-        return {
-            targetTableName,
-            refreshKeyValues: [],
-            lastUpdatedAt: versionEntry.last_updated_at,
-            buildRangeEnd: versionEntry.build_range_end,
-        };
-    }
-    updateLastTouch(tableName) {
-        this.preAggregations.updateLastTouch(tableName).catch(e => {
-            this.logger('Error on pre-aggregation touch', {
-                error: (e.stack || e), preAggregation: this.preAggregation, requestId: this.requestId,
-            });
-        });
-    }
-    contentVersion(invalidationKeys) {
-        const versionArray = [this.preAggregation.structureVersionLoadSql || this.preAggregation.loadSql];
-        if (this.preAggregation.indexesSql && this.preAggregation.indexesSql.length) {
-            versionArray.push(this.preAggregation.indexesSql);
-        }
-        if (this.preAggregation.streamOffset) {
-            versionArray.push(this.preAggregation.streamOffset);
-        }
-        if (this.preAggregation.outputColumnTypes) {
-            versionArray.push(this.preAggregation.outputColumnTypes);
-        }
-        versionArray.push(invalidationKeys);
-        return version(versionArray);
-    }
-    priority(defaultValue) {
-        return this.preAggregation.priority != null ? this.preAggregation.priority : defaultValue;
-    }
-    getInvalidationKeyValues() {
-        return Promise.all((this.preAggregation.invalidateKeyQueries || []).map((sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10))));
-    }
-    getPartitionInvalidationKeyValues() {
-        if (this.preAggregation.partitionInvalidateKeyQueries) {
-            return Promise.all((this.preAggregation.partitionInvalidateKeyQueries || []).map((sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10))));
-        }
-        else {
-            return this.getInvalidationKeyValues();
-        }
-    }
-    scheduleRefresh(invalidationKeys, newVersionEntry) {
-        this.logger('Refreshing pre-aggregation content', {
-            preAggregation: this.preAggregation,
-            requestId: this.requestId,
-            queryKey: this.preAggregationQueryKey(invalidationKeys),
-            newVersionEntry
-        });
-        this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry)
-            .catch(e => {
-            if (!(e instanceof ContinueWaitError_1.ContinueWaitError)) {
-                this.logger('Error refreshing pre-aggregation', {
-                    error: (e.stack || e), preAggregation: this.preAggregation, requestId: this.requestId
-                });
-            }
-        });
-    }
-    async executeInQueue(invalidationKeys, priority, newVersionEntry) {
-        const queue = await this.preAggregations.getQueue(this.preAggregation.dataSource);
-        return queue.executeInQueue('query', this.preAggregationQueryKey(invalidationKeys), {
-            preAggregation: this.preAggregation,
-            preAggregationsTablesToTempTables: this.preAggregationsTablesToTempTables,
-            newVersionEntry,
-            requestId: this.requestId,
-            invalidationKeys,
-            forceBuild: this.forceBuild,
-            isJob: this.isJob,
-            metadata: this.metadata,
-            orphanedTimeout: this.orphanedTimeout,
-        }, priority, 
-        // eslint-disable-next-line no-use-before-define
-        { stageQueryKey: PreAggregations.preAggregationQueryCacheKey(this.preAggregation), requestId: this.requestId });
-    }
-    preAggregationQueryKey(invalidationKeys) {
-        return this.preAggregation.indexesSql && this.preAggregation.indexesSql.length ?
-            [this.preAggregation.loadSql, this.preAggregation.indexesSql, invalidationKeys] :
-            [this.preAggregation.loadSql, invalidationKeys];
-    }
-    targetTableName(versionEntry) {
-        // eslint-disable-next-line no-use-before-define
-        return PreAggregations.targetTableName(versionEntry);
-    }
-    refresh(newVersionEntry, invalidationKeys, client) {
-        this.updateLastTouch(this.targetTableName(newVersionEntry));
-        let refreshStrategy = this.refreshStoreInSourceStrategy;
-        if (this.preAggregation.external) {
-            const readOnly = this.preAggregation.readOnly ||
-                client.config && client.config.readOnly ||
-                client.readOnly && (typeof client.readOnly === 'boolean' ? client.readOnly : client.readOnly());
-            if (readOnly) {
-                refreshStrategy = this.refreshReadOnlyExternalStrategy;
-            }
-            else {
-                refreshStrategy = this.refreshWriteStrategy;
-            }
-        }
-        return (0, base_driver_1.cancelCombinator)(saveCancelFn => refreshStrategy.bind(this)(client, newVersionEntry, saveCancelFn, invalidationKeys));
-    }
-    logExecutingSql(payload) {
-        this.logger('Executing Load Pre Aggregation SQL', payload);
-    }
-    queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry) {
-        return {
-            queryKey: this.preAggregationQueryKey(invalidationKeys),
-            query,
-            values: params,
-            targetTableName,
-            requestId: this.requestId,
-            newVersionEntry,
-            buildRangeEnd: this.preAggregation.buildRangeEnd,
-        };
-    }
-    async refreshStoreInSourceStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
-        const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
-        const targetTableName = this.targetTableName(newVersionEntry);
-        const query = QueryCache_1.QueryCache.replacePreAggregationTableNames(loadSql, this.preAggregationsTablesToTempTables).replace(this.preAggregation.tableName, targetTableName);
-        const queryOptions = this.queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry);
-        this.logExecutingSql(queryOptions);
-        try {
-            // TODO move index creation to the driver
-            await saveCancelFn(client.loadPreAggregationIntoTable(targetTableName, query, params, {
-                streamOffset: this.preAggregation.streamOffset,
-                outputColumnTypes: this.preAggregation.outputColumnTypes,
-                ...queryOptions
-            }));
-            await this.createIndexes(client, newVersionEntry, saveCancelFn, queryOptions);
-            await this.loadCache.fetchTables(this.preAggregation);
-        }
-        finally {
-            // We must clean orphaned in any cases: success or exception
-            await this.dropOrphanedTables(client, targetTableName, saveCancelFn, false, queryOptions);
-            await this.loadCache.fetchTables(this.preAggregation);
-        }
-    }
-    async refreshWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
-        const capabilities = client?.capabilities();
-        const withTempTable = !(capabilities?.unloadWithoutTempTable);
-        const dropSourceTempTable = !capabilities?.streamingSource;
-        return this.runWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable, dropSourceTempTable);
-    }
-    /**
-     * Runs export strategy with write access in data source
-     */
-    async runWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable, dropSourceTempTable) {
-        if (withTempTable) {
-            await client.createSchemaIfNotExists(this.preAggregation.preAggregationsSchema);
-        }
-        const targetTableName = this.targetTableName(newVersionEntry);
-        const queryOptions = await this.prepareWriteStrategy(client, targetTableName, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable);
-        try {
-            const tableData = await this.downloadExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions, withTempTable);
-            try {
-                await this.uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions);
-            }
-            finally {
-                if (tableData && tableData.release) {
-                    await tableData.release();
-                }
-            }
-        }
-        finally {
-            await this.cleanupWriteStrategy(client, targetTableName, queryOptions, saveCancelFn, withTempTable, dropSourceTempTable);
-        }
-    }
-    /**
-     * Cleanup tables after write strategy
-     */
-    async cleanupWriteStrategy(client, targetTableName, queryOptions, saveCancelFn, withTempTable, dropSourceTempTable) {
-        if (withTempTable && dropSourceTempTable) {
-            await this.withDropLock(false, async () => {
-                this.logger('Dropping source temp table', queryOptions);
-                const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema);
-                const mappedActualTables = actualTables.map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`);
-                if (mappedActualTables.includes(targetTableName)) {
-                    await client.dropTable(targetTableName);
-                }
-            });
-        }
-        // We must clean orphaned in any cases: success or exception
-        await this.loadCache.fetchTables(this.preAggregation);
-        await this.dropOrphanedTables(client, targetTableName, saveCancelFn, false, queryOptions);
-    }
-    /**
-     * Create table (if required) and prepares query options object
-     */
-    async prepareWriteStrategy(client, targetTableName, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable) {
-        if (withTempTable) {
-            const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
-            const query = QueryCache_1.QueryCache.replacePreAggregationTableNames(loadSql, this.preAggregationsTablesToTempTables).replace(this.preAggregation.tableName, targetTableName);
-            const queryOptions = this.queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry);
-            this.logExecutingSql(queryOptions);
-            await saveCancelFn(client.loadPreAggregationIntoTable(targetTableName, query, params, {
-                streamOffset: this.preAggregation.streamOffset,
-                outputColumnTypes: this.preAggregation.outputColumnTypes,
-                ...queryOptions
-            }));
-            return queryOptions;
-        }
-        else {
-            const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
-            const queryOptions = this.queryOptions(invalidationKeys, sql, params, targetTableName, newVersionEntry);
-            this.logExecutingSql(queryOptions);
-            return queryOptions;
-        }
-    }
-    /**
-     * Strategy to copy pre-aggregation from source db (for read-only permissions) to external data
-     */
-    async refreshReadOnlyExternalStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
-        const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
-        const queryOptions = this.queryOptions(invalidationKeys, sql, params, this.targetTableName(newVersionEntry), newVersionEntry);
-        this.logExecutingSql(queryOptions);
-        this.logger('Downloading external pre-aggregation via query', queryOptions);
-        const externalDriver = await this.externalDriverFactory();
-        const capabilities = externalDriver.capabilities && externalDriver.capabilities();
-        let tableData;
-        if (capabilities.csvImport && client.unloadFromQuery && await client.isUnloadSupported(this.getUnloadOptions())) {
-            tableData = await saveCancelFn(client.unloadFromQuery(sql, params, this.getUnloadOptions())).catch((error) => {
-                this.logger('Downloading external pre-aggregation via query error', { ...queryOptions, error: error.stack || error.message });
-                throw error;
-            });
-        }
-        else {
-            tableData = await saveCancelFn(client.downloadQueryResults(sql, params, {
-                streamOffset: this.preAggregation.streamOffset,
-                outputColumnTypes: this.preAggregation.outputColumnTypes,
-                ...queryOptions,
-                ...capabilities,
-                ...this.getStreamingOptions(),
-            })).catch((error) => {
-                this.logger('Downloading external pre-aggregation via query error', { ...queryOptions, error: error.stack || error.message });
-                throw error;
-            });
-        }
-        this.logger('Downloading external pre-aggregation via query completed', {
-            ...queryOptions,
-            isUnloadSupported: (0, base_driver_1.isDownloadTableCSVData)(tableData)
-        });
-        try {
-            await this.uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions);
-        }
-        finally {
-            if (tableData.release) {
-                await tableData.release();
-            }
-        }
-        await this.loadCache.fetchTables(this.preAggregation);
-    }
-    getUnloadOptions() {
-        return {
-            // Default: 16mb for Snowflake, Should be specified in MBs, because drivers convert it
-            maxFileSize: 64
-        };
-    }
-    getStreamingOptions() {
-        return {
-            // Default: 16384 (16KB), or 16 for objectMode streams. PostgreSQL/MySQL use object streams
-            highWaterMark: 10000
-        };
-    }
-    /**
-     * prepares download data for future cube store usage
-     */
-    async downloadExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions, withTempTable) {
-        const table = this.targetTableName(newVersionEntry);
-        this.logger('Downloading external pre-aggregation', queryOptions);
-        try {
-            const externalDriver = await this.externalDriverFactory();
-            const capabilities = externalDriver.capabilities && externalDriver.capabilities();
-            let tableData;
-            if (withTempTable) {
-                tableData = await this.getTableDataWithTempTable(client, table, saveCancelFn, queryOptions, capabilities);
-            }
-            else {
-                tableData = await this.getTableDataWithoutTempTable(client, table, saveCancelFn, queryOptions, capabilities);
-            }
-            this.logger('Downloading external pre-aggregation completed', {
-                ...queryOptions,
-                isUnloadSupported: (0, base_driver_1.isDownloadTableCSVData)(tableData)
-            });
-            return tableData;
-        }
-        catch (error) {
-            this.logger('Downloading external pre-aggregation error', {
-                ...queryOptions,
-                error: error?.stack || error?.message
-            });
-            throw error;
-        }
-    }
-    /**
-     * prepares download data when temp table = true
-     */
-    async getTableDataWithTempTable(client, table, saveCancelFn, queryOptions, externalDriverCapabilities) {
-        let tableData;
-        if (externalDriverCapabilities.csvImport && client.unload && await client.isUnloadSupported(this.getUnloadOptions())) {
-            tableData = await saveCancelFn(client.unload(table, this.getUnloadOptions()));
-        }
-        else if (externalDriverCapabilities.streamImport && client.stream) {
-            tableData = await saveCancelFn(client.stream(`SELECT * FROM ${table}`, [], this.getStreamingOptions()));
-            if (client.unload) {
-                const stream = new StreamObjectsCounter_1.LargeStreamWarning(this.preAggregation.preAggregationId, (msg) => {
-                    this.logger('Downloading external pre-aggregation warning', {
-                        ...queryOptions,
-                        error: msg
-                    });
-                });
-                tableData.rowStream.pipe(stream);
-                tableData.rowStream = stream;
-            }
-        }
-        else {
-            tableData = await saveCancelFn(client.downloadTable(table, {
-                streamOffset: this.preAggregation.streamOffset,
-                outputColumnTypes: this.preAggregation.outputColumnTypes,
-                ...externalDriverCapabilities
-            }));
-        }
-        if (!tableData.types) {
-            tableData.types = await saveCancelFn(client.tableColumnTypes(table));
-        }
-        return tableData;
-    }
-    /**
-     * prepares download data when temp table = false
-     */
-    async getTableDataWithoutTempTable(client, table, saveCancelFn, queryOptions, externalDriverCapabilities) {
-        const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
-        let tableData;
-        if (externalDriverCapabilities.csvImport && client.unload && await client.isUnloadSupported(this.getUnloadOptions())) {
-            return saveCancelFn(client.unload(table, { ...this.getUnloadOptions(), query: { sql, params } }));
-        }
-        else if (externalDriverCapabilities.streamImport && client.stream) {
-            tableData = await saveCancelFn(client.stream(sql, params, this.getStreamingOptions()));
-            if (client.unload) {
-                const stream = new StreamObjectsCounter_1.LargeStreamWarning(this.preAggregation.preAggregationId, (msg) => {
-                    this.logger('Downloading external pre-aggregation warning', {
-                        ...queryOptions,
-                        error: msg
-                    });
-                });
-                tableData.rowStream.pipe(stream);
-                tableData.rowStream = stream;
-            }
-        }
-        else {
-            tableData = { rows: await saveCancelFn(client.query(sql, params)) };
-        }
-        if (!tableData.types && client.queryColumnTypes) {
-            tableData.types = await saveCancelFn(client.queryColumnTypes(sql, params));
-        }
-        return tableData;
-    }
-    async uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions) {
-        const externalDriver = await this.externalDriverFactory();
-        const table = this.targetTableName(newVersionEntry);
-        this.logger('Uploading external pre-aggregation', queryOptions);
-        await saveCancelFn(externalDriver.uploadTableWithIndexes(table, tableData.types, tableData, this.prepareIndexesSql(newVersionEntry, queryOptions), this.preAggregation.uniqueKeyColumns, queryOptions, {
-            aggregationsColumns: this.preAggregation.aggregationsColumns,
-            createTableIndexes: this.prepareCreateTableIndexes(newVersionEntry),
-            sealAt: this.preAggregation.sealAt
-        })).catch((error) => {
-            this.logger('Uploading external pre-aggregation error', { ...queryOptions, error: error?.stack || error?.message });
-            throw error;
-        });
-        this.logger('Uploading external pre-aggregation completed', queryOptions);
-        await this.loadCache.fetchTables(this.preAggregation);
-        await this.dropOrphanedTables(externalDriver, table, saveCancelFn, true, queryOptions);
-    }
-    async createIndexes(driver, newVersionEntry, saveCancelFn, queryOptions) {
-        const indexesSql = this.prepareIndexesSql(newVersionEntry, queryOptions);
-        for (let i = 0; i < indexesSql.length; i++) {
-            const [query, params] = indexesSql[i].sql;
-            await saveCancelFn(driver.query(query, params));
-        }
-    }
-    prepareIndexesSql(newVersionEntry, queryOptions) {
-        if (!this.preAggregation.indexesSql || !this.preAggregation.indexesSql.length) {
-            return [];
-        }
-        return this.preAggregation.indexesSql.map(({ sql, indexName }) => {
-            const [query, params] = sql;
-            const indexVersionEntry = {
-                ...newVersionEntry,
-                table_name: indexName
-            };
-            this.logger('Creating pre-aggregation index', queryOptions);
-            const resultingSql = QueryCache_1.QueryCache.replacePreAggregationTableNames(query, this.preAggregationsTablesToTempTables.concat([
-                [this.preAggregation.tableName, { targetTableName: this.targetTableName(newVersionEntry) }],
-                [indexName, { targetTableName: this.targetTableName(indexVersionEntry) }]
-            ]));
-            return { sql: [resultingSql, params] };
-        });
-    }
-    prepareCreateTableIndexes(newVersionEntry) {
-        if (!this.preAggregation.createTableIndexes || !this.preAggregation.createTableIndexes.length) {
-            return [];
-        }
-        return this.preAggregation.createTableIndexes.map(({ indexName, type, columns }) => {
-            const indexVersionEntry = {
-                ...newVersionEntry,
-                table_name: indexName
-            };
-            return { indexName: this.targetTableName(indexVersionEntry), type, columns };
-        });
-    }
-    async withDropLock(external, lockFn) {
-        const lockKey = this.dropLockKey(external);
-        return this.queryCache.withLock(lockKey, 60 * 5, lockFn);
-    }
-    async dropOrphanedTables(client, justCreatedTable, saveCancelFn, external, queryOptions) {
-        await this.preAggregations.addTableUsed(justCreatedTable);
-        return this.withDropLock(external, async () => {
-            this.logger('Dropping orphaned tables', { ...queryOptions, external });
-            const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema);
-            const versionEntries = tablesToVersionEntries(this.preAggregation.preAggregationsSchema, actualTables);
-            const versionEntriesToSave = ramda_1.default.pipe(ramda_1.default.groupBy(v => v.table_name), ramda_1.default.toPairs, ramda_1.default.map(p => p[1][0]))(versionEntries);
-            const structureVersionsToSave = ramda_1.default.pipe(ramda_1.default.filter((v) => (new Date().getTime() - v.last_updated_at <
-                this.structureVersionPersistTime * 1000)), ramda_1.default.groupBy(v => `${v.table_name}_${v.structure_version}`), ramda_1.default.toPairs, ramda_1.default.map(p => p[1][0]))(versionEntries);
-            const refreshEndReached = await this.preAggregations.getRefreshEndReached();
-            const toSave = this.preAggregations.dropPreAggregationsWithoutTouch && refreshEndReached
-                ? (await this.preAggregations.tablesUsed())
-                    .concat(await this.preAggregations.tablesTouched())
-                    .concat([justCreatedTable])
-                : (await this.preAggregations.tablesUsed())
-                    .concat(structureVersionsToSave.map(v => this.targetTableName(v)))
-                    .concat(versionEntriesToSave.map(v => this.targetTableName(v)))
-                    .concat([justCreatedTable]);
-            const toDrop = actualTables
-                .map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`)
-                .filter(t => toSave.indexOf(t) === -1);
-            await Promise.all(toDrop.map(table => saveCancelFn(client.dropTable(table))));
-            this.logger('Dropping orphaned tables completed', {
-                ...queryOptions,
-                external,
-                tablesToDrop: JSON.stringify(toDrop),
-            });
-        });
-    }
-    dropLockKey(external) {
-        return external
-            ? 'drop-orphaned-tables-external'
-            : `drop-orphaned-tables:${this.preAggregation.dataSource}`;
-    }
-}
-exports.PreAggregationLoader = PreAggregationLoader;
-class PreAggregationPartitionRangeLoader {
-    redisPrefix;
-    driverFactory;
-    logger;
-    queryCache;
-    preAggregations;
-    preAggregation;
-    preAggregationsTablesToTempTables;
-    loadCache;
-    options;
-    /**
-     * Determines whether current instance instantiated for a jobed build query
-     * (initialized by the /cubejs-system/v1/pre-aggregations/jobs endpoint) or
-     * not.
-     */
-    isJob;
-    waitForRenew;
-    requestId;
-    lambdaQuery;
-    dataSource;
-    compilerCacheFn;
-    constructor(redisPrefix, driverFactory, logger, queryCache, 
-    // eslint-disable-next-line no-use-before-define
-    preAggregations, preAggregation, preAggregationsTablesToTempTables, loadCache, options = {
-        maxPartitions: 10000,
-        maxSourceRowLimit: 10000,
-    }) {
-        this.redisPrefix = redisPrefix;
-        this.driverFactory = driverFactory;
-        this.logger = logger;
-        this.queryCache = queryCache;
-        this.preAggregations = preAggregations;
-        this.preAggregation = preAggregation;
-        this.preAggregationsTablesToTempTables = preAggregationsTablesToTempTables;
-        this.loadCache = loadCache;
-        this.options = options;
-        this.isJob = !!options.isJob;
-        this.waitForRenew = options.waitForRenew;
-        this.requestId = options.requestId;
-        this.lambdaQuery = options.lambdaQuery;
-        this.dataSource = preAggregation.dataSource;
-        this.compilerCacheFn = options.compilerCacheFn || ((subKey, cacheFn) => cacheFn());
-    }
-    async loadRangeQuery(rangeQuery, partitionRange) {
-        const [query, values, queryOptions] = rangeQuery;
-        const invalidate = this.preAggregation.invalidateKeyQueries &&
-            this.preAggregation.invalidateKeyQueries[0]
-            ? this.preAggregation.invalidateKeyQueries[0].slice(0, 2)
-            : false;
-        return this.queryCache.cacheQueryResult(query, values, QueryCache_1.QueryCache.queryCacheKey({
-            query,
-            values: values,
-            invalidate,
-        }), 24 * 60 * 60, {
-            renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold
-                || queryOptions?.renewalThreshold || 24 * 60 * 60,
-            waitForRenew: this.waitForRenew,
-            priority: this.priority(10),
-            requestId: this.requestId,
-            dataSource: this.dataSource,
-            useInMemory: true,
-            external: queryOptions?.external,
-            renewalKey: partitionRange ? await this.getInvalidationKeyValues(partitionRange) : null,
-        });
-    }
-    getInvalidationKeyValues(range) {
-        const partitionTableName = PreAggregationPartitionRangeLoader.partitionTableName(this.preAggregation.tableName, this.preAggregation.partitionGranularity, range);
-        return Promise.all((this.preAggregation.invalidateKeyQueries || []).map((sqlQuery) => (this.loadCache.keyQueryResult(this.replacePartitionSqlAndParams(sqlQuery, range, partitionTableName), this.waitForRenew, this.priority(10)))));
-    }
-    priority(defaultValue) {
-        return this.preAggregation.priority != null ? this.preAggregation.priority : defaultValue;
-    }
-    async replaceQueryBuildRangeParams(queryValues) {
-        if (queryValues?.find(p => p === shared_1.BUILD_RANGE_START_LOCAL || p === shared_1.BUILD_RANGE_END_LOCAL)) {
-            const [buildRangeStart, buildRangeEnd] = await this.loadBuildRange();
-            return queryValues?.map(param => {
-                if (param === shared_1.BUILD_RANGE_START_LOCAL) {
-                    return (0, shared_1.utcToLocalTimeZone)(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
-                }
-                else if (param === shared_1.BUILD_RANGE_END_LOCAL) {
-                    return (0, shared_1.utcToLocalTimeZone)(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
-                }
-                else {
-                    return param;
-                }
-            });
-        }
-        return null;
-    }
-    replacePartitionSqlAndParams(query, dateRange, partitionTableName) {
-        const [sql, params, options] = query;
-        const updateWindowToBoundary = options?.incremental && (0, shared_1.addSecondsToLocalTimestamp)(dateRange[1], this.preAggregation.timezone, options?.updateWindowSeconds || 0);
-        return [sql.replace(this.preAggregation.tableName, partitionTableName), params?.map(param => {
-                if (dateRange && param === shared_1.FROM_PARTITION_RANGE) {
-                    return PreAggregationPartitionRangeLoader.inDbTimeZone(this.preAggregation, dateRange[0]);
-                }
-                else if (dateRange && param === shared_1.TO_PARTITION_RANGE) {
-                    return PreAggregationPartitionRangeLoader.inDbTimeZone(this.preAggregation, dateRange[1]);
-                }
-                else {
-                    return param;
-                }
-            }), {
-                ...options,
-                renewalThreshold: options?.incremental && updateWindowToBoundary < new Date() ?
-                    // if updateWindowToBoundary passed just moments ago we want to renew it earlier in case
-                    // of server and db clock don't match
-                    Math.min(Math.round((new Date().getTime() - updateWindowToBoundary.getTime()) / 1000), options?.renewalThresholdOutsideUpdateWindow) :
-                    options?.renewalThreshold
-            }];
-    }
-    partitionPreAggregationDescription(range, buildRange) {
-        const partitionTableName = PreAggregationPartitionRangeLoader.partitionTableName(this.preAggregation.tableName, this.preAggregation.partitionGranularity, range);
-        const [_, buildRangeEnd] = buildRange;
-        const loadRange = [...range];
-        const partitionInvalidateKeyQueries = this.preAggregation.partitionInvalidateKeyQueries || this.preAggregation.invalidateKeyQueries;
-        // `partitionInvalidateKeyQueries = []` in case of real time
-        if ((!partitionInvalidateKeyQueries || partitionInvalidateKeyQueries.length > 0) && buildRangeEnd < range[1]) {
-            loadRange[1] = buildRangeEnd;
-        }
-        const sealAt = (0, shared_1.addSecondsToLocalTimestamp)(loadRange[1], this.preAggregation.timezone, this.preAggregation.updateWindowSeconds || 0).toISOString();
-        return {
-            ...this.preAggregation,
-            tableName: partitionTableName,
-            structureVersionLoadSql: this.preAggregation.loadSql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.loadSql, range, partitionTableName),
-            loadSql: this.preAggregation.loadSql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.loadSql, loadRange, partitionTableName),
-            sql: this.preAggregation.sql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.sql, loadRange, partitionTableName),
-            invalidateKeyQueries: (this.preAggregation.invalidateKeyQueries || [])
-                .map(q => this.replacePartitionSqlAndParams(q, range, partitionTableName)),
-            partitionInvalidateKeyQueries: this.preAggregation.partitionInvalidateKeyQueries &&
-                this.preAggregation.partitionInvalidateKeyQueries.map(q => this.replacePartitionSqlAndParams(q, range, partitionTableName)),
-            indexesSql: (this.preAggregation.indexesSql || [])
-                .map(q => ({ ...q, sql: this.replacePartitionSqlAndParams(q.sql, range, partitionTableName) })),
-            previewSql: this.preAggregation.previewSql &&
-                this.replacePartitionSqlAndParams(this.preAggregation.previewSql, range, partitionTableName),
-            buildRangeStart: loadRange[0],
-            buildRangeEnd: loadRange[1],
-            sealAt, // Used only for kSql pre aggregations
-        };
-    }
-    async loadPreAggregations() {
-        if (this.preAggregation.partitionGranularity && !this.preAggregation.expandedPartition) {
-            const loadPreAggregationsByPartitionRanges = async ({ buildRange, partitionRanges }) => {
-                const partitionLoaders = partitionRanges.map(range => new PreAggregationLoader(this.redisPrefix, this.driverFactory, this.logger, this.queryCache, this.preAggregations, this.partitionPreAggregationDescription(range, buildRange), this.preAggregationsTablesToTempTables, this.loadCache, this.options));
-                const resolveResults = await Promise.all(partitionLoaders.map(async (l, i) => {
-                    const result = await l.loadPreAggregation(false);
-                    return result && {
-                        ...result,
-                        partitionRange: partitionRanges[i]
-                    };
-                }));
-                return { loadResults: resolveResults.filter(res => res !== null), partitionLoaders };
-            };
-            // eslint-disable-next-line prefer-const
-            let loadResultAndLoaders = await loadPreAggregationsByPartitionRanges(await this.partitionRanges());
-            if (this.options.externalRefresh && loadResultAndLoaders.loadResults.length === 0) {
-                loadResultAndLoaders = await loadPreAggregationsByPartitionRanges(await this.partitionRanges(true));
-                // In case there're no partitions ready at matched time dimension intersection then no data can be retrieved.
-                // We need to provide any table so query can just execute successfully.
-                if (loadResultAndLoaders.loadResults.length > 0) {
-                    loadResultAndLoaders.loadResults = [loadResultAndLoaders.loadResults[loadResultAndLoaders.loadResults.length - 1]];
-                }
-            }
-            if (this.options.externalRefresh && loadResultAndLoaders.loadResults.length === 0) {
-                throw new Error(
-                // eslint-disable-next-line no-use-before-define
-                PreAggregations.noPreAggregationPartitionsBuiltMessage(loadResultAndLoaders.partitionLoaders.map(p => p.preAggregation)));
-            }
-            let { loadResults } = loadResultAndLoaders;
-            let lambdaTable;
-            let emptyResult = false;
-            if (this.preAggregation.rollupLambdaId) {
-                if (this.lambdaQuery && loadResults.length > 0) {
-                    const { buildRangeEnd, targetTableName } = loadResults[loadResults.length - 1];
-                    const lambdaTypes = await this.loadCache.getTableColumnTypes(this.preAggregation, targetTableName);
-                    lambdaTable = await this.downloadLambdaTable(buildRangeEnd, lambdaTypes);
-                }
-                const rollupLambdaResults = this.preAggregationsTablesToTempTables.filter(tempTableResult => tempTableResult[1].rollupLambdaId === this.preAggregation.rollupLambdaId);
-                const filteredResults = loadResults.filter(r => (this.preAggregation.lastRollupLambda || (0, shared_1.reformatInIsoLocal)(r.buildRangeEnd) === (0, shared_1.reformatInIsoLocal)(r.partitionRange[1])) &&
-                    rollupLambdaResults.every(result => !result[1].buildRangeEnd || (0, shared_1.reformatInIsoLocal)(result[1].buildRangeEnd) < (0, shared_1.reformatInIsoLocal)(r.partitionRange[0])));
-                if (filteredResults.length === 0) {
-                    emptyResult = true;
-                    loadResults = [loadResults[loadResults.length - 1]];
-                }
-                else {
-                    loadResults = filteredResults;
-                }
-            }
-            const allTableTargetNames = loadResults.map(targetTableName => targetTableName.targetTableName);
-            let lastUpdatedAt = getLastUpdatedAtTimestamp(loadResults.map(r => r.lastUpdatedAt));
-            if (lambdaTable) {
-                allTableTargetNames.push(lambdaTable.name);
-                lastUpdatedAt = Date.now();
-            }
-            const unionTargetTableName = allTableTargetNames
-                .map(targetTableName => `SELECT * FROM ${targetTableName}${emptyResult ? ' WHERE 1 = 0' : ''}`)
-                .join(' UNION ALL ');
-            return {
-                targetTableName: allTableTargetNames.length === 1 && !emptyResult ? allTableTargetNames[0] : `(${unionTargetTableName})`,
-                refreshKeyValues: loadResults.map(t => t.refreshKeyValues),
-                lastUpdatedAt,
-                buildRangeEnd: !emptyResult && loadResults.length && loadResults[loadResults.length - 1].buildRangeEnd,
-                lambdaTable,
-                rollupLambdaId: this.preAggregation.rollupLambdaId,
-            };
-        }
-        else {
-            return new PreAggregationLoader(this.redisPrefix, this.driverFactory, this.logger, this.queryCache, this.preAggregations, this.preAggregation, this.preAggregationsTablesToTempTables, this.loadCache, this.options).loadPreAggregation(true);
-        }
-    }
-    /**
-     * Downloads the lambda table from the source DB.
-     */
-    async downloadLambdaTable(fromDate, lambdaTypes) {
-        const { sqlAndParams, cacheKeyQueries } = this.lambdaQuery;
-        const [query, params] = sqlAndParams;
-        const values = params.map((p) => {
-            if (p === shared_1.FROM_PARTITION_RANGE) {
-                return fromDate;
-            }
-            if (p === shared_1.MAX_SOURCE_ROW_LIMIT) {
-                return this.options.maxSourceRowLimit;
-            }
-            return p;
-        });
-        const { data } = await this.queryCache.renewQuery(query, values, cacheKeyQueries, 60 * 60, [query, values], undefined, {
-            requestId: this.requestId,
-            skipRefreshKeyWaitForRenew: false,
-            dataSource: this.dataSource,
-            external: false,
-            useCsvQuery: true,
-            lambdaTypes,
-        });
-        if (data.rowCount === this.options.maxSourceRowLimit) {
-            throw new Error(`The maximum number of source rows ${this.options.maxSourceRowLimit} was reached for ${this.preAggregation.preAggregationId}`);
-        }
-        return {
-            name: `${exports.LAMBDA_TABLE_PREFIX}_${this.preAggregation.tableName.replace('.', '_')}`,
-            columns: data.types,
-            csvRows: data.csvRows,
-        };
-    }
-    async partitionPreAggregations() {
-        if (this.preAggregation.partitionGranularity && !this.preAggregation.expandedPartition) {
-            const { buildRange, partitionRanges } = await this.partitionRanges();
-            return this.compilerCacheFn(['partitions', JSON.stringify(buildRange)], () => partitionRanges.map(range => this.partitionPreAggregationDescription(range, buildRange)));
-        }
-        else {
-            return [this.preAggregation];
-        }
-    }
-    async partitionRanges(ignoreMatchedDateRange) {
-        const buildRange = await this.loadBuildRange();
-        if (!buildRange[0] || !buildRange[1]) {
-            return { buildRange, partitionRanges: [] };
-        }
-        let dateRange = PreAggregationPartitionRangeLoader.intersectDateRanges(buildRange, ignoreMatchedDateRange ? undefined : this.preAggregation.matchedTimeDimensionDateRange);
-        if (!dateRange) {
-            // If there's no date range intersection between query data range and pre-aggregation build range
-            // use last partition so outer query can receive expected table structure.
-            dateRange = [buildRange[1], buildRange[1]];
-        }
-        const partitionRanges = this.compilerCacheFn(['timeSeries', this.preAggregation.partitionGranularity, JSON.stringify(dateRange), `${this.preAggregation.timestampPrecision}`], () => PreAggregationPartitionRangeLoader.timeSeries(this.preAggregation.partitionGranularity, dateRange, this.preAggregation.timestampPrecision));
-        if (partitionRanges.length > this.options.maxPartitions) {
-            throw new Error(`Pre-aggregation '${this.preAggregation.tableName}' requested to build ${partitionRanges.length} partitions which exceeds the maximum number of partitions per pre-aggregation of ${this.options.maxPartitions}`);
-        }
-        return { buildRange: dateRange, partitionRanges };
-    }
-    async loadBuildRange() {
-        const { preAggregationStartEndQueries } = this.preAggregation;
-        const [startDate, endDate] = await Promise.all(preAggregationStartEndQueries.map(async (rangeQuery) => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery))));
-        if (!this.preAggregation.partitionGranularity) {
-            return this.orNowIfEmpty([startDate, endDate]);
-        }
-        const wholeSeriesRanges = PreAggregationPartitionRangeLoader.timeSeries(this.preAggregation.partitionGranularity, this.orNowIfEmpty([startDate, endDate]), this.preAggregation.timestampPrecision);
-        const [rangeStart, rangeEnd] = await Promise.all(preAggregationStartEndQueries.map(async (rangeQuery, i) => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1]))));
-        return this.orNowIfEmpty([rangeStart, rangeEnd]);
-    }
-    now() {
-        return (0, shared_1.utcToLocalTimeZone)(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
-    }
-    orNowIfEmpty(dateRange) {
-        if (!dateRange[0] && !dateRange[1]) {
-            const now = this.now();
-            return [now, now];
-        }
-        if (!dateRange[0]) {
-            return [dateRange[1], dateRange[1]];
-        }
-        if (!dateRange[1]) {
-            return [dateRange[0], dateRange[0]];
-        }
-        return dateRange;
-    }
-    static checkDataRangeType(range) {
-        if (!range) {
-            return;
-        }
-        if (range.length !== 2) {
-            throw new Error(`Date range expected to be an array with 2 elements but ${range} found`);
-        }
-        if (typeof range[0] !== 'string' || typeof range[1] !== 'string') {
-            throw new Error(`Date range expected to be a string array but ${range} found`);
-        }
-        if ((range[0].length !== 23 && range[0].length !== 26) || (range[1].length !== 23 && range[0].length !== 26)) {
-            throw new Error(`Date range expected to be in YYYY-MM-DDTHH:mm:ss.SSS format but ${range} found`);
-        }
-    }
-    static intersectDateRanges(rangeA, rangeB) {
-        PreAggregationPartitionRangeLoader.checkDataRangeType(rangeA);
-        PreAggregationPartitionRangeLoader.checkDataRangeType(rangeB);
-        if (!rangeB) {
-            return rangeA;
-        }
-        if (!rangeA) {
-            return rangeB;
-        }
-        const from = rangeA[0] > rangeB[0] ? rangeA[0] : rangeB[0];
-        const to = rangeA[1] < rangeB[1] ? rangeA[1] : rangeB[1];
-        if (from > to) {
-            return null;
-        }
-        return [
-            from,
-            to,
-        ];
-    }
-    static timeSeries(granularity, dateRange, timestampPrecision) {
-        return (0, shared_1.timeSeries)(granularity, dateRange, {
-            timestampPrecision
-        });
-    }
-    static partitionTableName(tableName, partitionGranularity, dateRange) {
-        const partitionSuffix = dateRange[0].substring(0, partitionGranularity === 'hour' ? 13 : 10).replace(/[-T:]/g, '');
-        return `${tableName}${partitionSuffix}`;
-    }
-    static inDbTimeZone(preAggregationDescription, timestamp) {
-        return (0, shared_1.inDbTimeZone)(preAggregationDescription.timezone, preAggregationDescription.timestampFormat, timestamp);
-    }
-    static extractDate(data) {
-        return (0, shared_1.extractDate)(data);
-    }
-    static FROM_PARTITION_RANGE = shared_1.FROM_PARTITION_RANGE;
-    static TO_PARTITION_RANGE = shared_1.TO_PARTITION_RANGE;
-}
-exports.PreAggregationPartitionRangeLoader = PreAggregationPartitionRangeLoader;
+exports.tablesToVersionEntries = tablesToVersionEntries;
 class PreAggregations {
     redisPrefix;
     driverFactory;
@@ -1417,7 +174,7 @@ class PreAggregations {
      */
     async isPartitionExist(request, external, dataSource = 'default', schema, table, key, token) {
         // fetching tables
-        const loadCache = new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+        const loadCache = new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
             requestId: request,
             dataSource,
             tablePrefixes: external ? null : [schema],
@@ -1465,7 +222,7 @@ class PreAggregations {
         const getLoadCacheByDataSource = (dataSource = 'default', preAggregationSchema) => {
             if (!loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`]) {
                 loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`] =
-                    new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+                    new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
                         requestId: queryBody.requestId,
                         dataSource,
                         tablePrefixes: 
@@ -1480,7 +237,7 @@ class PreAggregations {
         };
         let queryParamsReplacement = null;
         const preAggregationsTablesToTempTablesPromise = preAggregations.map((p, i) => (preAggregationsTablesToTempTables) => {
-            const loader = new PreAggregationPartitionRangeLoader(this.redisPrefix, () => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, preAggregationsTablesToTempTables, getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
+            const loader = new PreAggregationPartitionRangeLoader_1.PreAggregationPartitionRangeLoader(() => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, preAggregationsTablesToTempTables, getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
                 maxPartitions: this.options.maxPartitions,
                 maxSourceRowLimit: this.options.maxSourceRowLimit,
                 isJob: queryBody.isJob,
@@ -1544,7 +301,7 @@ class PreAggregations {
         const getLoadCacheByDataSource = (dataSource = 'default', preAggregationSchema) => {
             if (!loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`]) {
                 loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`] =
-                    new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+                    new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
                         requestId: queryBody.requestId,
                         dataSource,
                     });
@@ -1552,7 +309,7 @@ class PreAggregations {
             return loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`];
         };
         const expandedPreAggregations = await Promise.all(preAggregations.map(p => {
-            const loader = new PreAggregationPartitionRangeLoader(this.redisPrefix, () => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, [], getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
+            const loader = new PreAggregationPartitionRangeLoader_1.PreAggregationPartitionRangeLoader(() => this.driverFactory(p.dataSource || 'default'), this.logger, this.queryCache, this, p, [], getLoadCacheByDataSource(p.dataSource, p.preAggregationsSchema), {
                 maxPartitions: this.options.maxPartitions,
                 maxSourceRowLimit: this.options.maxSourceRowLimit,
                 waitForRenew: queryBody.renewQuery,
@@ -1577,7 +334,7 @@ class PreAggregations {
             if (!this.queue[dataSource]) {
                 this.queue[dataSource] = QueryCache_1.QueryCache.createQueue(`SQL_PRE_AGGREGATIONS_${this.redisPrefix}_${dataSource}`, () => this.driverFactory(dataSource), (client, q) => {
                     const { preAggregation, preAggregationsTablesToTempTables, newVersionEntry, requestId, invalidationKeys, buildRangeEnd } = q;
-                    const loader = new PreAggregationLoader(this.redisPrefix, () => this.driverFactory(dataSource), this.logger, this.queryCache, this, preAggregation, preAggregationsTablesToTempTables, new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+                    const loader = new PreAggregationLoader_1.PreAggregationLoader(() => this.driverFactory(dataSource), this.logger, this.queryCache, this, preAggregation, preAggregationsTablesToTempTables, new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
                         requestId,
                         dataSource,
                     }), { requestId, externalRefresh: this.externalRefresh, buildRangeEnd });
@@ -1608,7 +365,7 @@ class PreAggregations {
             // eslint-disable-next-line @typescript-eslint/no-empty-function
             () => ({}), (_, q) => {
                 const { preAggregation, requestId } = q;
-                const loadCache = new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+                const loadCache = new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
                     requestId,
                     dataSource,
                 });
@@ -1655,7 +412,7 @@ class PreAggregations {
         const getLoadCacheByDataSource = (dataSource = 'default', preAggregationSchema) => {
             if (!loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`]) {
                 loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`] =
-                    new PreAggregationLoadCache(this.redisPrefix, () => this.driverFactory(dataSource), this.queryCache, this, {
+                    new PreAggregationLoadCache_1.PreAggregationLoadCache(() => this.driverFactory(dataSource), this.queryCache, this, {
                         requestId,
                         dataSource,
                     });
@@ -1677,8 +434,7 @@ class PreAggregations {
     }
     async getQueueState(dataSource) {
         const queue = await this.getQueue(dataSource);
-        const queries = await queue.getQueries();
-        return queries;
+        return queue.getQueries();
     }
     async cancelQueriesFromQueue(queryKeys, dataSource) {
         const queue = await this.getQueue(dataSource);