@cubejs-backend/query-orchestrator 1.2.13 → 1.2.15

package/dist/src/orchestrator/PreAggregationLoader.js

@@ -0,0 +1,742 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PreAggregationLoader = void 0;
+const ramda_1 = __importDefault(require("ramda"));
+const shared_1 = require("@cubejs-backend/shared");
+const base_driver_1 = require("@cubejs-backend/base-driver");
+const QueryCache_1 = require("./QueryCache");
+const ContinueWaitError_1 = require("./ContinueWaitError");
+const StreamObjectsCounter_1 = require("./StreamObjectsCounter");
+const PreAggregations_1 = require("./PreAggregations");
+// There are community developed and custom drivers which not always up-to-date with latest BaseDriver.
+// Extra defence for drivers that don't expose now() yet.
+function nowTimestamp(client) {
+    return client.nowTimestamp?.() ?? new Date().getTime();
+}
+class PreAggregationLoader {
+    driverFactory;
+    logger;
+    queryCache;
+    loadCache;
+    preAggregations;
+    preAggregation;
+    preAggregationsTablesToTempTables;
+    /**
+     * Determines whether current instance instantiated for a jobbed build query
+     * (initialized by the /cubejs-system/v1/pre-aggregations/jobs endpoint) or
+     * not.
+     */
+    isJob;
+    waitForRenew;
+    forceBuild;
+    orphanedTimeout;
+    externalDriverFactory;
+    requestId;
+    metadata;
+    structureVersionPersistTime;
+    externalRefresh;
+    constructor(driverFactory, logger, queryCache, preAggregations, preAggregation, preAggregationsTablesToTempTables, loadCache, options = {}) {
+        this.driverFactory = driverFactory;
+        this.logger = logger;
+        this.queryCache = queryCache;
+        this.loadCache = loadCache;
+        this.preAggregations = preAggregations;
+        this.preAggregation = preAggregation;
+        this.preAggregationsTablesToTempTables = preAggregationsTablesToTempTables;
+        this.isJob = !!options.isJob;
+        this.waitForRenew = options.waitForRenew;
+        this.forceBuild = options.forceBuild;
+        this.orphanedTimeout = options.orphanedTimeout;
+        this.externalDriverFactory = preAggregations.externalDriverFactory;
+        this.requestId = options.requestId;
+        this.metadata = options.metadata;
+        this.structureVersionPersistTime = preAggregations.structureVersionPersistTime;
+        this.externalRefresh = options.externalRefresh;
+        if (this.externalRefresh && this.waitForRenew) {
+            const message = 'Invalid configuration - when externalRefresh is true, it will not perform a renew, therefore you cannot wait for it using waitForRenew.';
+            if (['production', 'test'].includes((0, shared_1.getEnv)('nodeEnv'))) {
+                throw new Error(message);
+            }
+            else {
+                this.logger('Invalid Configuration', {
+                    requestId: this.requestId,
+                    warning: message,
+                });
+                this.waitForRenew = false;
+            }
+        }
+    }
+    async loadPreAggregation(throwOnMissingPartition) {
+        const notLoadedKey = (this.preAggregation.invalidateKeyQueries || [])
+            .find(keyQuery => !this.loadCache.hasKeyQueryResult(keyQuery));
+        if (this.isJob || !(notLoadedKey && !this.waitForRenew)) {
+            // Case 1: pre-agg build job processing.
+            // Case 2: either we have no data cached for this rollup or waitForRenew
+            // is true, either way, synchronously renew what data is needed so that
+            // the most current data will be returned fo the current request.
+            const result = await this.loadPreAggregationWithKeys();
+            const refreshKeyValues = await this.getInvalidationKeyValues();
+            return {
+                ...result,
+                refreshKeyValues,
+                queryKey: this.isJob
+                    // We need to return a queryKey value for the jobed build query
+                    // (initialized by the /cubejs-system/v1/pre-aggregations/jobs
+                    // endpoint) as a part of the response to make it possible to get a
+                    // query result from the cache by the other API call.
+                    ? this.preAggregationQueryKey(refreshKeyValues)
+                    : undefined,
+            };
+        }
+        else {
+            // Case 3: pre-agg exists
+            const structureVersion = (0, PreAggregations_1.getStructureVersion)(this.preAggregation);
+            const getVersionsStarted = new Date();
+            const { byStructure } = await this.loadCache.getVersionEntries(this.preAggregation);
+            this.logger('Load PreAggregations Tables', {
+                preAggregation: this.preAggregation,
+                requestId: this.requestId,
+                duration: (new Date().getTime() - getVersionsStarted.getTime())
+            });
+            const versionEntryByStructureVersion = byStructure[`${this.preAggregation.tableName}_${structureVersion}`];
+            if (this.externalRefresh) {
+                if (!versionEntryByStructureVersion && throwOnMissingPartition) {
+                    // eslint-disable-next-line no-use-before-define
+                    throw new Error(PreAggregations_1.PreAggregations.noPreAggregationPartitionsBuiltMessage([this.preAggregation]));
+                }
+                if (!versionEntryByStructureVersion) {
+                    return null;
+                }
+                else {
+                    // the rollups are being maintained independently of this instance of cube.js
+                    // immediately return the latest rollup data that instance already has
+                    return {
+                        targetTableName: this.targetTableName(versionEntryByStructureVersion),
+                        refreshKeyValues: [],
+                        lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
+                        buildRangeEnd: versionEntryByStructureVersion.build_range_end,
+                    };
+                }
+            }
+            if (versionEntryByStructureVersion) {
+                // this triggers an asynchronous/background load of the pre-aggregation but immediately
+                // returns the latest data it already has
+                this.loadPreAggregationWithKeys().catch(e => {
+                    if (!(e instanceof ContinueWaitError_1.ContinueWaitError)) {
+                        this.logger('Error loading pre-aggregation', {
+                            error: (e.stack || e),
+                            preAggregation: this.preAggregation,
+                            requestId: this.requestId
+                        });
+                    }
+                });
+                return {
+                    targetTableName: this.targetTableName(versionEntryByStructureVersion),
+                    refreshKeyValues: [],
+                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
+                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
+                };
+            }
+            else {
+                // no rollup has been built yet - build it synchronously as part of responding to this request
+                return this.loadPreAggregationWithKeys();
+            }
+        }
+    }
+    async loadPreAggregationWithKeys() {
+        const invalidationKeys = await this.getPartitionInvalidationKeyValues();
+        const contentVersion = this.contentVersion(invalidationKeys);
+        const structureVersion = (0, PreAggregations_1.getStructureVersion)(this.preAggregation);
+        const versionEntries = await this.loadCache.getVersionEntries(this.preAggregation);
+        const getVersionEntryByContentVersion = ({ byContent }) => byContent[`${this.preAggregation.tableName}_${contentVersion}`];
+        const versionEntryByContentVersion = getVersionEntryByContentVersion(versionEntries);
+        if (versionEntryByContentVersion && !this.forceBuild) {
+            const targetTableName = this.targetTableName(versionEntryByContentVersion);
+            // No need to block here
+            this.updateLastTouch(targetTableName);
+            return {
+                targetTableName,
+                refreshKeyValues: [],
+                lastUpdatedAt: versionEntryByContentVersion.last_updated_at,
+                buildRangeEnd: versionEntryByContentVersion.build_range_end,
+            };
+        }
+        if (!this.waitForRenew && !this.forceBuild) {
+            const versionEntryByStructureVersion = versionEntries.byStructure[`${this.preAggregation.tableName}_${structureVersion}`];
+            if (versionEntryByStructureVersion) {
+                const targetTableName = this.targetTableName(versionEntryByStructureVersion);
+                // No need to block here
+                this.updateLastTouch(targetTableName);
+                return {
+                    targetTableName,
+                    refreshKeyValues: [],
+                    lastUpdatedAt: versionEntryByStructureVersion.last_updated_at,
+                    buildRangeEnd: versionEntryByStructureVersion.build_range_end,
+                };
+            }
+        }
+        const client = this.preAggregation.external ?
+            await this.externalDriverFactory() :
+            await this.driverFactory();
+        if (!versionEntries.versionEntries.length) {
+            await client.createSchemaIfNotExists(this.preAggregation.preAggregationsSchema);
+        }
+        // ensure we find appropriate structure version before invalidating anything
+        const versionEntry = versionEntries.byStructure[`${this.preAggregation.tableName}_${structureVersion}`] ||
+            versionEntries.byTableName[this.preAggregation.tableName];
+        const newVersionEntry = {
+            table_name: this.preAggregation.tableName,
+            structure_version: structureVersion,
+            content_version: contentVersion,
+            last_updated_at: nowTimestamp(client),
+            naming_version: 2,
+        };
+        const mostRecentResult = async () => {
+            await this.loadCache.reset(this.preAggregation);
+            const lastVersion = getVersionEntryByContentVersion(await this.loadCache.getVersionEntries(this.preAggregation));
+            if (!lastVersion) {
+                throw new Error(`Pre-aggregation table is not found for ${this.preAggregation.tableName} after it was successfully created`);
+            }
+            const targetTableName = this.targetTableName(lastVersion);
+            this.updateLastTouch(targetTableName);
+            return {
+                targetTableName,
+                refreshKeyValues: [],
+                lastUpdatedAt: lastVersion.last_updated_at,
+                buildRangeEnd: lastVersion.build_range_end,
+            };
+        };
+        if (this.forceBuild) {
+            this.logger('Force build pre-aggregation', {
+                preAggregation: this.preAggregation,
+                requestId: this.requestId,
+                metadata: this.metadata,
+                queryKey: this.preAggregationQueryKey(invalidationKeys),
+                newVersionEntry
+            });
+            if (this.isJob) {
+                // We don't want to wait for the jobed build query result. So we run the
+                // executeInQueue method and immediately return the LoadPreAggregationResult object.
+                this
+                    .executeInQueue(invalidationKeys, this.priority(10), newVersionEntry)
+                    .catch((e) => {
+                    this.logger('Pre-aggregations build job error', {
+                        preAggregation: this.preAggregation,
+                        requestId: this.requestId,
+                        newVersionEntry,
+                        error: (e.stack || e),
+                    });
+                });
+                const targetTableName = this.targetTableName(newVersionEntry);
+                this.updateLastTouch(targetTableName);
+                return {
+                    targetTableName,
+                    refreshKeyValues: [],
+                    lastUpdatedAt: newVersionEntry.last_updated_at,
+                    buildRangeEnd: this.preAggregation.buildRangeEnd,
+                };
+            }
+            else {
+                await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
+                return mostRecentResult();
+            }
+        }
+        if (versionEntry) {
+            if (versionEntry.structure_version !== newVersionEntry.structure_version) {
+                this.logger('Invalidating pre-aggregation structure', {
+                    preAggregation: this.preAggregation,
+                    requestId: this.requestId,
+                    queryKey: this.preAggregationQueryKey(invalidationKeys),
+                    newVersionEntry
+                });
+                await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
+                return mostRecentResult();
+            }
+            else if (versionEntry.content_version !== newVersionEntry.content_version) {
+                if (this.waitForRenew) {
+                    this.logger('Waiting for pre-aggregation renew', {
+                        preAggregation: this.preAggregation,
+                        requestId: this.requestId,
+                        queryKey: this.preAggregationQueryKey(invalidationKeys),
+                        newVersionEntry
+                    });
+                    await this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry);
+                    return mostRecentResult();
+                }
+                else {
+                    this.scheduleRefresh(invalidationKeys, newVersionEntry);
+                }
+            }
+        }
+        else {
+            this.logger('Creating pre-aggregation from scratch', {
+                preAggregation: this.preAggregation,
+                requestId: this.requestId,
+                queryKey: this.preAggregationQueryKey(invalidationKeys),
+                newVersionEntry
+            });
+            await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
+            return mostRecentResult();
+        }
+        const targetTableName = this.targetTableName(versionEntry);
+        this.updateLastTouch(targetTableName);
+        return {
+            targetTableName,
+            refreshKeyValues: [],
+            lastUpdatedAt: versionEntry.last_updated_at,
+            buildRangeEnd: versionEntry.build_range_end,
+        };
+    }
+    updateLastTouch(tableName) {
+        this.preAggregations.updateLastTouch(tableName).catch(e => {
+            this.logger('Error on pre-aggregation touch', {
+                error: (e.stack || e), preAggregation: this.preAggregation, requestId: this.requestId,
+            });
+        });
+    }
+    contentVersion(invalidationKeys) {
+        const versionArray = [this.preAggregation.structureVersionLoadSql || this.preAggregation.loadSql];
+        if (this.preAggregation.indexesSql && this.preAggregation.indexesSql.length) {
+            versionArray.push(this.preAggregation.indexesSql);
+        }
+        if (this.preAggregation.streamOffset) {
+            versionArray.push(this.preAggregation.streamOffset);
+        }
+        if (this.preAggregation.outputColumnTypes) {
+            versionArray.push(this.preAggregation.outputColumnTypes);
+        }
+        versionArray.push(invalidationKeys);
+        return (0, PreAggregations_1.version)(versionArray);
+    }
+    priority(defaultValue) {
+        return this.preAggregation.priority != null ? this.preAggregation.priority : defaultValue;
+    }
+    getInvalidationKeyValues() {
+        return Promise.all((this.preAggregation.invalidateKeyQueries || []).map((sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10))));
+    }
+    getPartitionInvalidationKeyValues() {
+        if (this.preAggregation.partitionInvalidateKeyQueries) {
+            return Promise.all((this.preAggregation.partitionInvalidateKeyQueries || []).map((sqlQuery) => this.loadCache.keyQueryResult(sqlQuery, this.waitForRenew, this.priority(10))));
+        }
+        else {
+            return this.getInvalidationKeyValues();
+        }
+    }
+    scheduleRefresh(invalidationKeys, newVersionEntry) {
+        this.logger('Refreshing pre-aggregation content', {
+            preAggregation: this.preAggregation,
+            requestId: this.requestId,
+            queryKey: this.preAggregationQueryKey(invalidationKeys),
+            newVersionEntry
+        });
+        this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry)
+            .catch(e => {
+            if (!(e instanceof ContinueWaitError_1.ContinueWaitError)) {
+                this.logger('Error refreshing pre-aggregation', {
+                    error: (e.stack || e), preAggregation: this.preAggregation, requestId: this.requestId
+                });
+            }
+        });
+    }
+    async executeInQueue(invalidationKeys, priority, newVersionEntry) {
+        const queue = await this.preAggregations.getQueue(this.preAggregation.dataSource);
+        return queue.executeInQueue('query', this.preAggregationQueryKey(invalidationKeys), {
+            preAggregation: this.preAggregation,
+            preAggregationsTablesToTempTables: this.preAggregationsTablesToTempTables,
+            newVersionEntry,
+            requestId: this.requestId,
+            invalidationKeys,
+            forceBuild: this.forceBuild,
+            isJob: this.isJob,
+            metadata: this.metadata,
+            orphanedTimeout: this.orphanedTimeout,
+        }, priority, 
+        // eslint-disable-next-line no-use-before-define
+        { stageQueryKey: PreAggregations_1.PreAggregations.preAggregationQueryCacheKey(this.preAggregation), requestId: this.requestId });
+    }
+    preAggregationQueryKey(invalidationKeys) {
+        return this.preAggregation.indexesSql && this.preAggregation.indexesSql.length ?
+            [this.preAggregation.loadSql, this.preAggregation.indexesSql, invalidationKeys] :
+            [this.preAggregation.loadSql, invalidationKeys];
+    }
+    targetTableName(versionEntry) {
+        // eslint-disable-next-line no-use-before-define
+        return PreAggregations_1.PreAggregations.targetTableName(versionEntry);
+    }
+    refresh(newVersionEntry, invalidationKeys, client) {
+        this.updateLastTouch(this.targetTableName(newVersionEntry));
+        let refreshStrategy = this.refreshStoreInSourceStrategy;
+        if (this.preAggregation.external) {
+            const readOnly = this.preAggregation.readOnly ||
+                client.config && client.config.readOnly ||
+                client.readOnly && (typeof client.readOnly === 'boolean' ? client.readOnly : client.readOnly());
+            if (readOnly) {
+                refreshStrategy = this.refreshReadOnlyExternalStrategy;
+            }
+            else {
+                refreshStrategy = this.refreshWriteStrategy;
+            }
+        }
+        return (0, base_driver_1.cancelCombinator)(saveCancelFn => refreshStrategy.bind(this)(client, newVersionEntry, saveCancelFn, invalidationKeys));
+    }
+    logExecutingSql(payload) {
+        this.logger('Executing Load Pre Aggregation SQL', payload);
+    }
+    queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry) {
+        return {
+            queryKey: this.preAggregationQueryKey(invalidationKeys),
+            query,
+            values: params,
+            targetTableName,
+            requestId: this.requestId,
+            newVersionEntry,
+            buildRangeEnd: this.preAggregation.buildRangeEnd,
+        };
+    }
+    async refreshStoreInSourceStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
+        const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
+        const targetTableName = this.targetTableName(newVersionEntry);
+        const query = QueryCache_1.QueryCache.replacePreAggregationTableNames(loadSql, this.preAggregationsTablesToTempTables).replace(this.preAggregation.tableName, targetTableName);
+        const queryOptions = this.queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry);
+        this.logExecutingSql(queryOptions);
+        try {
+            // TODO move index creation to the driver
+            await saveCancelFn(client.loadPreAggregationIntoTable(targetTableName, query, params, {
+                streamOffset: this.preAggregation.streamOffset,
+                outputColumnTypes: this.preAggregation.outputColumnTypes,
+                ...queryOptions
+            }));
+            await this.createIndexes(client, newVersionEntry, saveCancelFn, queryOptions);
+            await this.loadCache.fetchTables(this.preAggregation);
+        }
+        finally {
+            // We must clean orphaned in any cases: success or exception
+            await this.dropOrphanedTables(client, targetTableName, saveCancelFn, false, queryOptions);
+            await this.loadCache.fetchTables(this.preAggregation);
+        }
+    }
+    async refreshWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
+        const capabilities = client?.capabilities();
+        const withTempTable = !(capabilities?.unloadWithoutTempTable);
+        const dropSourceTempTable = !capabilities?.streamingSource;
+        return this.runWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable, dropSourceTempTable);
+    }
+    /**
+     * Runs export strategy with write access in data source
+     */
+    async runWriteStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable, dropSourceTempTable) {
+        if (withTempTable) {
+            await client.createSchemaIfNotExists(this.preAggregation.preAggregationsSchema);
+        }
+        const targetTableName = this.targetTableName(newVersionEntry);
+        const queryOptions = await this.prepareWriteStrategy(client, targetTableName, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable);
+        try {
+            const tableData = await this.downloadExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions, withTempTable);
+            try {
+                await this.uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions);
+            }
+            finally {
+                if (tableData && tableData.release) {
+                    await tableData.release();
+                }
+            }
+        }
+        finally {
+            await this.cleanupWriteStrategy(client, targetTableName, queryOptions, saveCancelFn, withTempTable, dropSourceTempTable);
+        }
+    }
+    /**
+     * Cleanup tables after write strategy
+     */
+    async cleanupWriteStrategy(client, targetTableName, queryOptions, saveCancelFn, withTempTable, dropSourceTempTable) {
+        if (withTempTable && dropSourceTempTable) {
+            await this.withDropLock(false, async () => {
+                this.logger('Dropping source temp table', queryOptions);
+                const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema);
+                const mappedActualTables = actualTables.map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`);
+                if (mappedActualTables.includes(targetTableName)) {
+                    await client.dropTable(targetTableName);
+                }
+            });
+        }
+        // We must clean orphaned in any cases: success or exception
+        await this.loadCache.fetchTables(this.preAggregation);
+        await this.dropOrphanedTables(client, targetTableName, saveCancelFn, false, queryOptions);
+    }
+    /**
+     * Create table (if required) and prepares query options object
+     */
+    async prepareWriteStrategy(client, targetTableName, newVersionEntry, saveCancelFn, invalidationKeys, withTempTable) {
+        if (withTempTable) {
+            const [loadSql, params] = Array.isArray(this.preAggregation.loadSql) ? this.preAggregation.loadSql : [this.preAggregation.loadSql, []];
+            const query = QueryCache_1.QueryCache.replacePreAggregationTableNames(loadSql, this.preAggregationsTablesToTempTables).replace(this.preAggregation.tableName, targetTableName);
+            const queryOptions = this.queryOptions(invalidationKeys, query, params, targetTableName, newVersionEntry);
+            this.logExecutingSql(queryOptions);
+            await saveCancelFn(client.loadPreAggregationIntoTable(targetTableName, query, params, {
+                streamOffset: this.preAggregation.streamOffset,
+                outputColumnTypes: this.preAggregation.outputColumnTypes,
+                ...queryOptions
+            }));
+            return queryOptions;
+        }
+        else {
+            const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
+            const queryOptions = this.queryOptions(invalidationKeys, sql, params, targetTableName, newVersionEntry);
+            this.logExecutingSql(queryOptions);
+            return queryOptions;
+        }
+    }
+    /**
+     * Strategy to copy pre-aggregation from source db (for read-only permissions) to external data
+     */
+    async refreshReadOnlyExternalStrategy(client, newVersionEntry, saveCancelFn, invalidationKeys) {
+        const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
+        const queryOptions = this.queryOptions(invalidationKeys, sql, params, this.targetTableName(newVersionEntry), newVersionEntry);
+        this.logExecutingSql(queryOptions);
+        this.logger('Downloading external pre-aggregation via query', queryOptions);
+        const externalDriver = await this.externalDriverFactory();
+        const capabilities = externalDriver.capabilities && externalDriver.capabilities();
+        let tableData;
+        if (capabilities.csvImport && client.unloadFromQuery && await client.isUnloadSupported(this.getUnloadOptions())) {
+            tableData = await saveCancelFn(client.unloadFromQuery(sql, params, this.getUnloadOptions())).catch((error) => {
+                this.logger('Downloading external pre-aggregation via query error', {
+                    ...queryOptions,
+                    error: error.stack || error.message
+                });
+                throw error;
+            });
+        }
+        else {
+            tableData = await saveCancelFn(client.downloadQueryResults(sql, params, {
+                streamOffset: this.preAggregation.streamOffset,
+                outputColumnTypes: this.preAggregation.outputColumnTypes,
+                ...queryOptions,
+                ...capabilities,
+                ...this.getStreamingOptions(),
+            })).catch((error) => {
+                this.logger('Downloading external pre-aggregation via query error', {
+                    ...queryOptions,
+                    error: error.stack || error.message
+                });
+                throw error;
+            });
+        }
+        this.logger('Downloading external pre-aggregation via query completed', {
+            ...queryOptions,
+            isUnloadSupported: (0, base_driver_1.isDownloadTableCSVData)(tableData)
+        });
+        try {
+            await this.uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions);
+        }
+        finally {
+            if (tableData.release) {
+                await tableData.release();
+            }
+        }
+        await this.loadCache.fetchTables(this.preAggregation);
+    }
+    getUnloadOptions() {
+        return {
+            // Default: 16mb for Snowflake, Should be specified in MBs, because drivers convert it
+            maxFileSize: 64
+        };
+    }
+    getStreamingOptions() {
+        return {
+            // Default: 16384 (16KB), or 16 for objectMode streams. PostgreSQL/MySQL use object streams
+            highWaterMark: 10000
+        };
+    }
+    /**
+     * prepares download data for future cube store usage
+     */
+    async downloadExternalPreAggregation(client, newVersionEntry, saveCancelFn, queryOptions, withTempTable) {
+        const table = this.targetTableName(newVersionEntry);
+        this.logger('Downloading external pre-aggregation', queryOptions);
+        try {
+            const externalDriver = await this.externalDriverFactory();
+            const capabilities = externalDriver.capabilities && externalDriver.capabilities();
+            let tableData;
+            if (withTempTable) {
+                tableData = await this.getTableDataWithTempTable(client, table, saveCancelFn, queryOptions, capabilities);
+            }
+            else {
+                tableData = await this.getTableDataWithoutTempTable(client, table, saveCancelFn, queryOptions, capabilities);
+            }
+            this.logger('Downloading external pre-aggregation completed', {
+                ...queryOptions,
+                isUnloadSupported: (0, base_driver_1.isDownloadTableCSVData)(tableData)
+            });
+            return tableData;
+        }
+        catch (error) {
+            this.logger('Downloading external pre-aggregation error', {
+                ...queryOptions,
+                error: error?.stack || error?.message
+            });
+            throw error;
+        }
+    }
+    /**
+     * prepares download data when temp table = true
+     */
+    async getTableDataWithTempTable(client, table, saveCancelFn, queryOptions, externalDriverCapabilities) {
+        let tableData;
+        if (externalDriverCapabilities.csvImport && client.unload && await client.isUnloadSupported(this.getUnloadOptions())) {
+            tableData = await saveCancelFn(client.unload(table, this.getUnloadOptions()));
+        }
+        else if (externalDriverCapabilities.streamImport && client.stream) {
+            tableData = await saveCancelFn(client.stream(`SELECT * FROM ${table}`, [], this.getStreamingOptions()));
+            if (client.unload) {
+                const stream = new StreamObjectsCounter_1.LargeStreamWarning(this.preAggregation.preAggregationId, (msg) => {
+                    this.logger('Downloading external pre-aggregation warning', {
+                        ...queryOptions,
+                        error: msg
+                    });
+                });
+                tableData.rowStream.pipe(stream);
+                tableData.rowStream = stream;
+            }
+        }
+        else {
+            tableData = await saveCancelFn(client.downloadTable(table, {
+                streamOffset: this.preAggregation.streamOffset,
+                outputColumnTypes: this.preAggregation.outputColumnTypes,
+                ...externalDriverCapabilities
+            }));
+        }
+        if (!tableData.types) {
+            tableData.types = await saveCancelFn(client.tableColumnTypes(table));
+        }
+        return tableData;
+    }
+    /**
+     * prepares download data when temp table = false
+     */
+    async getTableDataWithoutTempTable(client, table, saveCancelFn, queryOptions, externalDriverCapabilities) {
+        const [sql, params] = Array.isArray(this.preAggregation.sql) ? this.preAggregation.sql : [this.preAggregation.sql, []];
+        let tableData;
+        if (externalDriverCapabilities.csvImport && client.unload && await client.isUnloadSupported(this.getUnloadOptions())) {
+            return saveCancelFn(client.unload(table, { ...this.getUnloadOptions(), query: { sql, params } }));
+        }
+        else if (externalDriverCapabilities.streamImport && client.stream) {
+            tableData = await saveCancelFn(client.stream(sql, params, this.getStreamingOptions()));
+            if (client.unload) {
+                const stream = new StreamObjectsCounter_1.LargeStreamWarning(this.preAggregation.preAggregationId, (msg) => {
+                    this.logger('Downloading external pre-aggregation warning', {
+                        ...queryOptions,
+                        error: msg
+                    });
+                });
+                tableData.rowStream.pipe(stream);
+                tableData.rowStream = stream;
+            }
+        }
+        else {
+            tableData = { rows: await saveCancelFn(client.query(sql, params)) };
+        }
+        if (!tableData.types && client.queryColumnTypes) {
+            tableData.types = await saveCancelFn(client.queryColumnTypes(sql, params));
+        }
+        return tableData;
+    }
+    async uploadExternalPreAggregation(tableData, newVersionEntry, saveCancelFn, queryOptions) {
+        const externalDriver = await this.externalDriverFactory();
+        const table = this.targetTableName(newVersionEntry);
+        this.logger('Uploading external pre-aggregation', queryOptions);
+        await saveCancelFn(externalDriver.uploadTableWithIndexes(table, tableData.types, tableData, this.prepareIndexesSql(newVersionEntry, queryOptions), this.preAggregation.uniqueKeyColumns, queryOptions, {
+            aggregationsColumns: this.preAggregation.aggregationsColumns,
+            createTableIndexes: this.prepareCreateTableIndexes(newVersionEntry),
+            sealAt: this.preAggregation.sealAt
+        })).catch((error) => {
+            this.logger('Uploading external pre-aggregation error', {
+                ...queryOptions,
+                error: error?.stack || error?.message
+            });
+            throw error;
+        });
+        this.logger('Uploading external pre-aggregation completed', queryOptions);
+        await this.loadCache.fetchTables(this.preAggregation);
+        await this.dropOrphanedTables(externalDriver, table, saveCancelFn, true, queryOptions);
+    }
+    async createIndexes(driver, newVersionEntry, saveCancelFn, queryOptions) {
+        const indexesSql = this.prepareIndexesSql(newVersionEntry, queryOptions);
+        for (let i = 0; i < indexesSql.length; i++) {
+            const [query, params] = indexesSql[i].sql;
+            await saveCancelFn(driver.query(query, params));
+        }
+    }
+    prepareIndexesSql(newVersionEntry, queryOptions) {
+        if (!this.preAggregation.indexesSql || !this.preAggregation.indexesSql.length) {
+            return [];
+        }
+        return this.preAggregation.indexesSql.map(({ sql, indexName }) => {
+            const [query, params] = sql;
+            const indexVersionEntry = {
+                ...newVersionEntry,
+                table_name: indexName
+            };
+            this.logger('Creating pre-aggregation index', queryOptions);
+            const resultingSql = QueryCache_1.QueryCache.replacePreAggregationTableNames(query, this.preAggregationsTablesToTempTables.concat([
+                [this.preAggregation.tableName, { targetTableName: this.targetTableName(newVersionEntry) }],
+                [indexName, { targetTableName: this.targetTableName(indexVersionEntry) }]
+            ]));
+            return { sql: [resultingSql, params] };
+        });
+    }
+    prepareCreateTableIndexes(newVersionEntry) {
+        if (!this.preAggregation.createTableIndexes || !this.preAggregation.createTableIndexes.length) {
+            return [];
+        }
+        return this.preAggregation.createTableIndexes.map(({ indexName, type, columns }) => {
+            const indexVersionEntry = {
+                ...newVersionEntry,
+                table_name: indexName
+            };
+            return { indexName: this.targetTableName(indexVersionEntry), type, columns };
+        });
+    }
+    async withDropLock(external, lockFn) {
+        const lockKey = this.dropLockKey(external);
+        return this.queryCache.withLock(lockKey, 60 * 5, lockFn);
+    }
+    async dropOrphanedTables(client, justCreatedTable, saveCancelFn, external, queryOptions) {
+        await this.preAggregations.addTableUsed(justCreatedTable);
+        return this.withDropLock(external, async () => {
+            this.logger('Dropping orphaned tables', { ...queryOptions, external });
+            const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema);
+            const versionEntries = (0, PreAggregations_1.tablesToVersionEntries)(this.preAggregation.preAggregationsSchema, actualTables);
+            const versionEntriesToSave = ramda_1.default.pipe(ramda_1.default.groupBy(v => v.table_name), ramda_1.default.toPairs, ramda_1.default.map(p => p[1][0]))(versionEntries);
+            const structureVersionsToSave = ramda_1.default.pipe(ramda_1.default.filter((v) => (new Date().getTime() - v.last_updated_at <
+                this.structureVersionPersistTime * 1000)), ramda_1.default.groupBy(v => `${v.table_name}_${v.structure_version}`), ramda_1.default.toPairs, ramda_1.default.map(p => p[1][0]))(versionEntries);
+            const refreshEndReached = await this.preAggregations.getRefreshEndReached();
+            const toSave = this.preAggregations.dropPreAggregationsWithoutTouch && refreshEndReached
+                ? (await this.preAggregations.tablesUsed())
+                    .concat(await this.preAggregations.tablesTouched())
+                    .concat([justCreatedTable])
+                : (await this.preAggregations.tablesUsed())
+                    .concat(structureVersionsToSave.map(v => this.targetTableName(v)))
+                    .concat(versionEntriesToSave.map(v => this.targetTableName(v)))
+                    .concat([justCreatedTable]);
+            const toDrop = actualTables
+                .map(t => `${this.preAggregation.preAggregationsSchema}.${t.table_name || t.TABLE_NAME}`)
+                .filter(t => toSave.indexOf(t) === -1);
+            await Promise.all(toDrop.map(table => saveCancelFn(client.dropTable(table))));
+            this.logger('Dropping orphaned tables completed', {
+                ...queryOptions,
+                external,
+                tablesToDrop: JSON.stringify(toDrop),
+            });
+        });
+    }
+    dropLockKey(external) {
+        return external
+            ? 'drop-orphaned-tables-external'
+            : `drop-orphaned-tables:${this.preAggregation.dataSource}`;
+    }
+}
+exports.PreAggregationLoader = PreAggregationLoader;
+//# sourceMappingURL=PreAggregationLoader.js.map