@cubejs-backend/query-orchestrator 1.2.13 → 1.2.14

package/dist/src/orchestrator/PreAggregationLoadCache.d.ts

@@ -0,0 +1,42 @@
+import { TableStructure } from '@cubejs-backend/base-driver';
+import { DriverFactory } from './DriverFactory';
+import { QueryWithParams } from './QueryCache';
+import { PreAggregationDescription, TableCacheEntry, VersionEntriesObj } from './PreAggregations';
+type PreAggregationLoadCacheOptions = {
+    requestId?: string;
+    dataSource: string;
+    tablePrefixes?: string[];
+};
+export declare class PreAggregationLoadCache {
+    private readonly driverFactory;
+    private queryCache;
+    private preAggregations;
+    private readonly queryResults;
+    private readonly externalDriverFactory;
+    private readonly requestId;
+    private versionEntries;
+    private tables;
+    private tableColumnTypes;
+    private queryStageState;
+    private readonly dataSource;
+    private readonly tablePrefixes;
+    constructor(clientFactory: DriverFactory, queryCache: any, preAggregations: any, options?: PreAggregationLoadCacheOptions);
+    protected tablesFromCache(preAggregation: any, forceRenew?: boolean): Promise<any>;
+    fetchTables(preAggregation: PreAggregationDescription): Promise<any>;
+    private fetchTablesNoCache;
+    tablesCachePrefixKey(preAggregation: PreAggregationDescription): string;
+    protected getTablesQuery(preAggregation: any): Promise<TableCacheEntry[]>;
+    getTableColumnTypes(preAggregation: PreAggregationDescription, tableName: string): Promise<TableStructure>;
+    private calculateVersionEntries;
+    getVersionEntries(preAggregation: any): Promise<VersionEntriesObj>;
+    keyQueryResult(sqlQuery: QueryWithParams, waitForRenew: boolean, priority: number): Promise<any>;
+    hasKeyQueryResult(keyQuery: any): boolean;
+    getQueryStage(stageQueryKey: any): Promise<{
+        stage: string;
+        timeElapsed: number;
+    }>;
+    protected fetchQueryStageState(queue?: any): Promise<any>;
+    reset(preAggregation: any): Promise<void>;
+}
+export {};
+//# sourceMappingURL=PreAggregationLoadCache.d.ts.map

package/dist/src/orchestrator/PreAggregationLoadCache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PreAggregationLoadCache.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/PreAggregationLoadCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAA0B,eAAe,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EACL,yBAAyB,EAEzB,eAAe,EAEf,iBAAiB,EAElB,MAAM,mBAAmB,CAAC;AAE3B,KAAK,8BAA8B,GAAG;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B,CAAC;AAEF,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAE9C,OAAO,CAAC,UAAU,CAAa;IAE/B,OAAO,CAAC,eAAe,CAAkB;IAEzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAM;IAEnC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAM;IAE5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEhC,OAAO,CAAC,cAAc,CAAqD;IAE3E,OAAO,CAAC,MAAM,CAA4C;IAE1D,OAAO,CAAC,gBAAgB,CAAkE;IAK1F,OAAO,CAAC,eAAe,CAAM;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;gBAG9C,aAAa,EAAE,aAAa,EAC5B,UAAU,KAAA,EACV,eAAe,KAAA,EACf,OAAO,GAAE,8BAA0D;cAerD,eAAe,CAAC,cAAc,KAAA,EAAE,UAAU,GAAE,OAAe;IAgB9D,WAAW,CAAC,cAAc,EAAE,yBAAyB;YAcpD,kBAAkB;IAUzB,oBAAoB,CAAC,cAAc,EAAE,yBAAyB;cAIrD,cAAc,CAAC,cAAc,KAAA;IAchC,mBAAmB,CAAC,cAAc,EAAE,yBAAyB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;YAgBzG,uBAAuB;IAqCxB,iBAAiB,CAAC,cAAc,KAAA,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAc7D,cAAc,CAAC,QAAQ,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAyBvF,iBAAiB,CAAC,QAAQ,KAAA;IAIpB,aAAa,CAAC,aAAa,KAAA;;;;cAMxB,oBAAoB,CAAC,KAAK,CAAC,KAAA;IAQ9B,KAAK,CAAC,cAAc,KAAA;CAOlC"}

package/dist/src/orchestrator/PreAggregationLoadCache.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PreAggregationLoadCache = void 0;
+const PreAggregations_1 = require("./PreAggregations");
+class PreAggregationLoadCache {
+    driverFactory;
+    queryCache;
+    preAggregations;
+    queryResults;
+    externalDriverFactory;
+    requestId;
+    versionEntries;
+    tables;
+    tableColumnTypes;
+    // TODO this is in memory cache structure as well however it depends on
+    // data source only and load cache is per data source for now.
+    // Make it per data source key in case load cache scope is broaden.
+    queryStageState;
+    dataSource;
+    tablePrefixes;
+    constructor(clientFactory, queryCache, preAggregations, options = { dataSource: 'default' }) {
+        this.dataSource = options.dataSource;
+        this.driverFactory = clientFactory;
+        this.queryCache = queryCache;
+        this.preAggregations = preAggregations;
+        this.queryResults = {};
+        this.externalDriverFactory = preAggregations.externalDriverFactory;
+        this.requestId = options.requestId;
+        this.tablePrefixes = options.tablePrefixes;
+        this.versionEntries = {};
+        this.tables = {};
+        this.tableColumnTypes = {};
+    }
+    async tablesFromCache(preAggregation, forceRenew = false) {
+        let tables = forceRenew ? null : await this.queryCache.getCacheDriver().get(this.tablesCachePrefixKey(preAggregation));
+        if (!tables) {
+            tables = await this.preAggregations.getLoadCacheQueue(this.dataSource).executeInQueue('query', `Fetch tables for ${preAggregation.preAggregationsSchema}`, {
+                preAggregation, requestId: this.requestId
+            }, 0, { requestId: this.requestId });
+        }
+        return tables;
+    }
+    async fetchTables(preAggregation) {
+        if (preAggregation.external && !this.externalDriverFactory) {
+            throw new Error('externalDriverFactory is not provided. Please use CUBEJS_DEV_MODE=true or provide Cube Store connection env variables for production usage.');
+        }
+        const newTables = await this.fetchTablesNoCache(preAggregation);
+        await this.queryCache.getCacheDriver().set(this.tablesCachePrefixKey(preAggregation), newTables, this.preAggregations.options.preAggregationsSchemaCacheExpire || 60 * 60);
+        return newTables;
+    }
+    async fetchTablesNoCache(preAggregation) {
+        const client = preAggregation.external ?
+            await this.externalDriverFactory() :
+            await this.driverFactory();
+        if (this.tablePrefixes && client.getPrefixTablesQuery && this.preAggregations.options.skipExternalCacheAndQueue) {
+            return client.getPrefixTablesQuery(preAggregation.preAggregationsSchema, this.tablePrefixes);
+        }
+        return client.getTablesQuery(preAggregation.preAggregationsSchema);
+    }
+    tablesCachePrefixKey(preAggregation) {
+        return this.queryCache.getKey('SQL_PRE_AGGREGATIONS_TABLES', `${preAggregation.dataSource}${preAggregation.preAggregationsSchema}${preAggregation.external ? '_EXT' : ''}`);
+    }
+    async getTablesQuery(preAggregation) {
+        const redisKey = this.tablesCachePrefixKey(preAggregation);
+        if (!this.tables[redisKey]) {
+            const tables = this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external ?
+                await this.fetchTablesNoCache(preAggregation) :
+                await this.tablesFromCache(preAggregation);
+            if (tables === undefined) {
+                throw new Error('Pre-aggregation tables are undefined.');
+            }
+            this.tables[redisKey] = tables;
+        }
+        return this.tables[redisKey];
+    }
+    async getTableColumnTypes(preAggregation, tableName) {
+        const prefixKey = this.tablesCachePrefixKey(preAggregation);
+        if (!this.tableColumnTypes[prefixKey]?.[tableName]) {
+            if (!this.preAggregations.options.skipExternalCacheAndQueue && preAggregation.external) {
+                throw new Error(`Lambda union with source data feature is supported only by external rollups stored in Cube Store but was invoked for '${preAggregation.preAggregationId}'`);
+            }
+            const client = await this.externalDriverFactory();
+            const columnTypes = await client.tableColumnTypes(tableName);
+            if (!this.tableColumnTypes[prefixKey]) {
+                this.tableColumnTypes[prefixKey] = {};
+            }
+            this.tableColumnTypes[prefixKey][tableName] = columnTypes;
+        }
+        return this.tableColumnTypes[prefixKey][tableName];
+    }
+    async calculateVersionEntries(preAggregation) {
+        let versionEntries = (0, PreAggregations_1.tablesToVersionEntries)(preAggregation.preAggregationsSchema, await this.getTablesQuery(preAggregation));
+        // It presumes strong consistency guarantees for external pre-aggregation tables ingestion
+        if (!preAggregation.external) {
+            const [, , queries] = await this.fetchQueryStageState();
+            const targetTableNamesInQueue = (Object.keys(queries))
+                .map(q => PreAggregations_1.PreAggregations.targetTableName(queries[q].query.newVersionEntry));
+            versionEntries = versionEntries.filter(e => targetTableNamesInQueue.indexOf(PreAggregations_1.PreAggregations.targetTableName(e)) === -1);
+        }
+        const byContent = {};
+        const byStructure = {};
+        const byTableName = {};
+        versionEntries.forEach(e => {
+            const contentKey = `${e.table_name}_${e.content_version}`;
+            if (!byContent[contentKey]) {
+                byContent[contentKey] = e;
+            }
+            const structureKey = `${e.table_name}_${e.structure_version}`;
+            if (!byStructure[structureKey]) {
+                byStructure[structureKey] = e;
+            }
+            if (!byTableName[e.table_name]) {
+                byTableName[e.table_name] = e;
+            }
+        });
+        return { versionEntries, byContent, byStructure, byTableName };
+    }
+    async getVersionEntries(preAggregation) {
+        if (this.tablePrefixes && !this.tablePrefixes.find(p => preAggregation.tableName.split('.')[1].startsWith(p))) {
+            throw new Error(`Load cache tries to load table ${preAggregation.tableName} outside of tablePrefixes filter: ${this.tablePrefixes.join(', ')}`);
+        }
+        const redisKey = this.tablesCachePrefixKey(preAggregation);
+        if (!this.versionEntries[redisKey]) {
+            this.versionEntries[redisKey] = this.calculateVersionEntries(preAggregation).catch(e => {
+                delete this.versionEntries[redisKey];
+                throw e;
+            });
+        }
+        return this.versionEntries[redisKey];
+    }
+    async keyQueryResult(sqlQuery, waitForRenew, priority) {
+        const [query, values, queryOptions] = Array.isArray(sqlQuery) ? sqlQuery : [sqlQuery, [], {}];
+        if (!this.queryResults[this.queryCache.queryRedisKey([query, values])]) {
+            this.queryResults[this.queryCache.queryRedisKey([query, values])] = await this.queryCache.cacheQueryResult(query, values, [query, values], 60 * 60, {
+                renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold
+                    || queryOptions?.renewalThreshold || 2 * 60,
+                renewalKey: [query, values],
+                waitForRenew,
+                priority,
+                requestId: this.requestId,
+                dataSource: this.dataSource,
+                useInMemory: true,
+                external: queryOptions?.external
+            });
+        }
+        return this.queryResults[this.queryCache.queryRedisKey([query, values])];
+    }
+    hasKeyQueryResult(keyQuery) {
+        return !!this.queryResults[this.queryCache.queryRedisKey(keyQuery)];
+    }
+    async getQueryStage(stageQueryKey) {
+        const queue = await this.preAggregations.getQueue(this.dataSource);
+        await this.fetchQueryStageState(queue);
+        return queue.getQueryStage(stageQueryKey, undefined, this.queryStageState);
+    }
+    async fetchQueryStageState(queue) {
+        queue = queue || await this.preAggregations.getQueue(this.dataSource);
+        if (!this.queryStageState) {
+            this.queryStageState = await queue.fetchQueryStageState();
+        }
+        return this.queryStageState;
+    }
+    async reset(preAggregation) {
+        await this.tablesFromCache(preAggregation, true);
+        this.tables = {};
+        this.tableColumnTypes = {};
+        this.queryStageState = undefined;
+        this.versionEntries = {};
+    }
+}
+exports.PreAggregationLoadCache = PreAggregationLoadCache;
+//# sourceMappingURL=PreAggregationLoadCache.js.map

package/dist/src/orchestrator/PreAggregationLoadCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PreAggregationLoadCache.js","sourceRoot":"","sources":["../../../src/orchestrator/PreAggregationLoadCache.ts"],"names":[],"mappings":";;;AAGA,uDAO2B;AAQ3B,MAAa,uBAAuB;IACjB,aAAa,CAAgB;IAEtC,UAAU,CAAa;IAEvB,eAAe,CAAkB;IAExB,YAAY,CAAM;IAElB,qBAAqB,CAAM;IAE3B,SAAS,CAAM;IAExB,cAAc,CAAqD;IAEnE,MAAM,CAA4C;IAElD,gBAAgB,CAAkE;IAE1F,uEAAuE;IACvE,8DAA8D;IAC9D,mEAAmE;IAC3D,eAAe,CAAM;IAEZ,UAAU,CAAS;IAEnB,aAAa,CAAkB;IAEhD,YACE,aAA4B,EAC5B,UAAU,EACV,eAAe,EACf,UAA0C,EAAE,UAAU,EAAE,SAAS,EAAE;QAEnE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,qBAAqB,GAAG,eAAe,CAAC,qBAAqB,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;IAC7B,CAAC;IAES,KAAK,CAAC,eAAe,CAAC,cAAc,EAAE,aAAsB,KAAK;QACzE,IAAI,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC,CAAC;QACvH,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,cAAc,CACnF,OAAO,EACP,oBAAoB,cAAc,CAAC,qBAAqB,EAAE,EAC1D;gBACE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1C,EACD,CAAC,EACD,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAC9B,CAAC;SACH;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,cAAyC;QAChE,IAAI,cAAc,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC1D,MAAM,IAAI,KAAK,CAAC,6IAA6I,CAAC,CAAC;SAChK;QAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,GAAG,CACxC,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,EACzC,SAAS,EACT,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,gCAAgC,IAAI,EAAE,GAAG,EAAE,CACzE,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,cAAyC;QACxE,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC;YACpC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,oBAAoB,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,yBAAyB,EAAE;YAC/G,OAAO,MAAM,CAAC,oBAAoB,CAAC,cAAc,CAAC,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;SAC9F;QACD,OAAO,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;IACrE,CAAC;IAEM,oBAAoB,CAAC,cAAyC;QACnE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,6BAA6B,EAAE,GAAG,cAAc,CAAC,UAAU,GAAG,cAAc,CAAC,qBAAqB,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9K,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,cAAc;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,yBAAyB,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAChG,MAAM,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;YAC7C,IAAI,MAAM,KAAK,SAAS,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;aAC1D;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;SAChC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,cAAyC,EAAE,SAAiB;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE;YAClD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,yBAAyB,IAAI,cAAc,CAAC,QAAQ,EAAE;gBACtF,MAAM,IAAI,KAAK,CAAC,yHAAyH,cAAc,CAAC,gBAAgB,GAAG,CAAC,CAAC;aAC9K;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAClD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC7D,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE;gBACrC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;aACvC;YACD,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,GAAG,WAAW,CAAC;SAC3D;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,cAAc;QAClD,IAAI,cAAc,GAAG,IAAA,wCAAsB,EACzC,cAAc,CAAC,qBAAqB,EACpC,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAC1C,CAAC;QACF,0FAA0F;QAC1F,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE;YAC5B,MAAM,CAAC,EAAC,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvD,MAAM,uBAAuB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACnD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,iCAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAE/E,cAAc,GAAG,cAAc,CAAC,MAAM,CACpC,CAAC,CAAC,EAAE,CAAC,uBAAuB,CAAC,OAAO,CAAC,iCAAe,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAChF,CAAC;SACH;QAED,MAAM,SAAS,GAAoC,EAAE,CAAC;QACtD,MAAM,WAAW,GAAoC,EAAE,CAAC;QACxD,MAAM,WAAW,GAAoC,EAAE,CAAC;QAExD,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACzB,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC;YAC1D,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE;gBAC1B,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aAC3B;YACD,MAAM,YAAY,GAAG,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,iBAAiB,EAAE,CAAC;YAC9D,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE;gBAC9B,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;aAC/B;YACD,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE;gBAC9B,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aAC/B;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;IACjE,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,cAAc;QAC3C,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE;YAC7G,MAAM,IAAI,KAAK,CAAC,kCAAkC,cAAc,CAAC,SAAS,qCAAqC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACjJ;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAClC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;gBACrF,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACrC,MAAM,CAAC,CAAC;YACV,CAAC,CAAC,CAAC;SACJ;QACD,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEM,KAAK,CAAC,cAAc,CAAC,QAAyB,EAAE,YAAqB,EAAE,QAAgB;QAC5F,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,GAAe,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1G,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE;YACtE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,CACxG,KAAK,EACK,MAAM,EAChB,CAAC,KAAK,EAAY,MAAM,CAAC,EACzB,EAAE,GAAG,EAAE,EACP;gBACE,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,0BAA0B;uBAC/D,YAAY,EAAE,gBAAgB,IAAI,CAAC,GAAG,EAAE;gBAC7C,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;gBAC3B,YAAY;gBACZ,QAAQ;gBACR,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,IAAI;gBACjB,QAAQ,EAAE,YAAY,EAAE,QAAQ;aACjC,CACF,CAAC;SACH;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IAEM,iBAAiB,CAAC,QAAQ;QAC/B,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtE,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,aAAa;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7E,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,KAAM;QACzC,KAAK,GAAG,KAAK,IAAI,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,IAAI,CAAC,eAAe,GAAG,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;SAC3D;QACD,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,cAAc;QAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;CACF;AA9ND,0DA8NC"}

package/dist/src/orchestrator/PreAggregationLoader.d.ts

@@ -0,0 +1,105 @@
+import { DownloadTableData, DriverCapabilities, DriverInterface, SaveCancelFn, StreamOptions, UnloadOptions } from '@cubejs-backend/base-driver';
+import { DriverFactory } from './DriverFactory';
+import { QueryCache, QueryTuple } from './QueryCache';
+import { InvalidationKeys, LoadPreAggregationResult, PreAggregations, VersionEntry } from './PreAggregations';
+import { PreAggregationLoadCache } from './PreAggregationLoadCache';
+type IndexesSql = {
+    sql: [string, unknown[]];
+    indexName: string;
+}[];
+type QueryKey = [QueryTuple, IndexesSql, InvalidationKeys] | [QueryTuple, InvalidationKeys];
+type QueryOptions = {
+    queryKey: QueryKey;
+    newVersionEntry: VersionEntry;
+    query: string;
+    values: unknown[];
+    requestId: string;
+    buildRangeEnd?: string;
+};
+export declare class PreAggregationLoader {
+    private readonly driverFactory;
+    private readonly logger;
+    private readonly queryCache;
+    private readonly loadCache;
+    private preAggregations;
+    preAggregation: any;
+    private preAggregationsTablesToTempTables;
+    /**
+     * Determines whether current instance instantiated for a jobbed build query
+     * (initialized by the /cubejs-system/v1/pre-aggregations/jobs endpoint) or
+     * not.
+     */
+    private readonly isJob;
+    private readonly waitForRenew;
+    private readonly forceBuild;
+    private readonly orphanedTimeout;
+    private readonly externalDriverFactory;
+    private readonly requestId;
+    private readonly metadata;
+    private readonly structureVersionPersistTime;
+    private readonly externalRefresh;
+    constructor(driverFactory: DriverFactory, logger: any, queryCache: QueryCache, preAggregations: PreAggregations, preAggregation: any, preAggregationsTablesToTempTables: any, loadCache: PreAggregationLoadCache, options?: any);
+    loadPreAggregation(throwOnMissingPartition: boolean): Promise<null | LoadPreAggregationResult>;
+    protected loadPreAggregationWithKeys(): Promise<LoadPreAggregationResult>;
+    private updateLastTouch;
+    protected contentVersion(invalidationKeys: InvalidationKeys): string;
+    protected priority(defaultValue: number): number;
+    protected getInvalidationKeyValues(): Promise<any[]>;
+    protected getPartitionInvalidationKeyValues(): Promise<any[]>;
+    protected scheduleRefresh(invalidationKeys: InvalidationKeys, newVersionEntry: VersionEntry): void;
+    protected executeInQueue(invalidationKeys: InvalidationKeys, priority: number, newVersionEntry: VersionEntry): Promise<any>;
+    protected preAggregationQueryKey(invalidationKeys: InvalidationKeys): QueryKey;
+    protected targetTableName(versionEntry: VersionEntry): string;
+    refresh(newVersionEntry: VersionEntry, invalidationKeys: InvalidationKeys, client: any): any;
+    protected logExecutingSql(payload: any): void;
+    protected queryOptions(invalidationKeys: InvalidationKeys, query: string, params: unknown[], targetTableName: string, newVersionEntry: VersionEntry): {
+        queryKey: QueryKey;
+        query: string;
+        values: unknown[];
+        targetTableName: string;
+        requestId: string;
+        newVersionEntry: VersionEntry;
+        buildRangeEnd: any;
+    };
+    protected refreshStoreInSourceStrategy(client: DriverInterface, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, invalidationKeys: InvalidationKeys): Promise<void>;
+    protected refreshWriteStrategy(client: DriverInterface, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, invalidationKeys: InvalidationKeys): Promise<void>;
+    /**
+     * Runs export strategy with write access in data source
+     */
+    protected runWriteStrategy(client: DriverInterface, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, invalidationKeys: InvalidationKeys, withTempTable: boolean, dropSourceTempTable: boolean): Promise<void>;
+    /**
+     * Cleanup tables after write strategy
+     */
+    protected cleanupWriteStrategy(client: DriverInterface, targetTableName: string, queryOptions: QueryOptions, saveCancelFn: SaveCancelFn, withTempTable: boolean, dropSourceTempTable: boolean): Promise<void>;
+    /**
+     * Create table (if required) and prepares query options object
+     */
+    protected prepareWriteStrategy(client: DriverInterface, targetTableName: string, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, invalidationKeys: InvalidationKeys, withTempTable: boolean): Promise<QueryOptions>;
+    /**
+     * Strategy to copy pre-aggregation from source db (for read-only permissions) to external data
+     */
+    protected refreshReadOnlyExternalStrategy(client: DriverInterface, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, invalidationKeys: InvalidationKeys): Promise<void>;
+    protected getUnloadOptions(): UnloadOptions;
+    protected getStreamingOptions(): StreamOptions;
+    /**
+     * prepares download data for future cube store usage
+     */
+    protected downloadExternalPreAggregation(client: DriverInterface, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, queryOptions: QueryOptions, withTempTable: boolean): Promise<import("@cubejs-backend/base-driver").StreamTableData | import("@cubejs-backend/base-driver").TableMemoryData | import("@cubejs-backend/base-driver").TableCSVData>;
+    /**
+     * prepares download data when temp table = true
+     */
+    protected getTableDataWithTempTable(client: DriverInterface, table: string, saveCancelFn: SaveCancelFn, queryOptions: QueryOptions, externalDriverCapabilities: DriverCapabilities): Promise<import("@cubejs-backend/base-driver").StreamTableData | import("@cubejs-backend/base-driver").TableMemoryData | import("@cubejs-backend/base-driver").TableCSVData>;
+    /**
+     * prepares download data when temp table = false
+     */
+    protected getTableDataWithoutTempTable(client: DriverInterface, table: string, saveCancelFn: SaveCancelFn, queryOptions: QueryOptions, externalDriverCapabilities: DriverCapabilities): Promise<import("@cubejs-backend/base-driver").StreamTableData | import("@cubejs-backend/base-driver").TableMemoryData | import("@cubejs-backend/base-driver").TableCSVData>;
+    protected uploadExternalPreAggregation(tableData: DownloadTableData, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, queryOptions: QueryOptions): Promise<void>;
+    protected createIndexes(driver: any, newVersionEntry: VersionEntry, saveCancelFn: SaveCancelFn, queryOptions: QueryOptions): Promise<void>;
+    protected prepareIndexesSql(newVersionEntry: VersionEntry, queryOptions: QueryOptions): any;
+    protected prepareCreateTableIndexes(newVersionEntry: VersionEntry): any;
+    private withDropLock;
+    protected dropOrphanedTables(client: DriverInterface, justCreatedTable: string, saveCancelFn: SaveCancelFn, external: boolean, queryOptions: QueryOptions): Promise<boolean>;
+    private dropLockKey;
+}
+export {};
+//# sourceMappingURL=PreAggregationLoader.d.ts.map

package/dist/src/orchestrator/PreAggregationLoader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PreAggregationLoader.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/PreAggregationLoader.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EAEf,YAAY,EACZ,aAAa,EACb,aAAa,EACd,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAGtD,OAAO,EAEL,gBAAgB,EAChB,wBAAwB,EACxB,eAAe,EAIf,YAAY,EACb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,KAAK,UAAU,GAAG;IAAE,GAAG,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC;AAEpE,KAAK,QAAQ,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;AAE5F,KAAK,YAAY,GAAG;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,eAAe,EAAE,YAAY,CAAC;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAQF,qBAAa,oBAAoB;IA+B7B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAI3B,OAAO,CAAC,QAAQ,CAAC,SAAS;IApC5B,OAAO,CAAC,eAAe,CAAkB;IAElC,cAAc,EAAE,GAAG,CAAC;IAE3B,OAAO,CAAC,iCAAiC,CAAM;IAE/C;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAEhC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAU;IAEvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IAErC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IAEzC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAgB;IAEtD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;IAE/B,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAM;IAElD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAU;gBAGvB,aAAa,EAAE,aAAa,EAC5B,MAAM,EAAE,GAAG,EACX,UAAU,EAAE,UAAU,EACvC,eAAe,EAAE,eAAe,EAChC,cAAc,KAAA,EACd,iCAAiC,KAAA,EAChB,SAAS,EAAE,uBAAuB,EACnD,OAAO,GAAE,GAAQ;IA6BN,kBAAkB,CAC7B,uBAAuB,EAAE,OAAO,GAC/B,OAAO,CAAC,IAAI,GAAG,wBAAwB,CAAC;cA8E3B,0BAA0B,IAAI,OAAO,CAAC,wBAAwB,CAAC;IA4J/E,OAAO,CAAC,eAAe;IAQvB,SAAS,CAAC,cAAc,CAAC,gBAAgB,EAAE,gBAAgB;IAe3D,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM;IAIhD,SAAS,CAAC,wBAAwB;IAQlC,SAAS,CAAC,iCAAiC;IAY3C,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,eAAe,EAAE,YAAY;cAiB3E,cAAc,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,YAAY;IAsBlH,SAAS,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,QAAQ;IAM9E,SAAS,CAAC,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,MAAM;IAKtD,OAAO,CAAC,eAAe,EAAE,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,KAAA;IAyBxF,SAAS,CAAC,eAAe,CAAC,OAAO,KAAA;IAOjC,SAAS,CAAC,YAAY,CAAC,gBAAgB,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,YAAY;;;;;;;;;cAYnI,4BAA4B,CAC1C,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB;cAuCpB,oBAAoB,CAClC,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB;IAiBpC;;OAEG;cACa,gBAAgB,CAC9B,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB,EAClC,aAAa,EAAE,OAAO,EACtB,mBAAmB,EAAE,OAAO;IA2C9B;;OAEG;cACa,oBAAoB,CAClC,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,MAAM,EACvB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,EAC1B,aAAa,EAAE,OAAO,EACtB,mBAAmB,EAAE,OAAO;IAmB9B;;OAEG;cACa,oBAAoB,CAClC,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,MAAM,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB,EAClC,aAAa,EAAE,OAAO,GACrB,OAAO,CAAC,YAAY,CAAC;IAqCxB;;OAEG;cACa,+BAA+B,CAC7C,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,gBAAgB,EAAE,gBAAgB;IA8DpC,SAAS,CAAC,gBAAgB,IAAI,aAAa;IAO3C,SAAS,CAAC,mBAAmB,IAAI,aAAa;IAO9C;;OAEG;cACa,8BAA8B,CAC5C,MAAM,EAAE,eAAe,EACvB,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,EAC1B,aAAa,EAAE,OAAO;IA+BxB;;OAEG;cACa,yBAAyB,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,0BAA0B,EAAE,kBAAkB;IAqCxL;;OAEG;cACa,4BAA4B,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,0BAA0B,EAAE,kBAAkB;cAuC3K,4BAA4B,CAC1C,SAAS,EAAE,iBAAiB,EAC5B,eAAe,EAAE,YAAY,EAC7B,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY;cAiCZ,aAAa,CAAC,MAAM,KAAA,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY;IAQ3H,SAAS,CAAC,iBAAiB,CAAC,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY;IAsBrF,SAAS,CAAC,yBAAyB,CAAC,eAAe,EAAE,YAAY;YAanD,YAAY;cAKV,kBAAkB,CAChC,MAAM,EAAE,eAAe,EACvB,gBAAgB,EAAE,MAAM,EACxB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,OAAO,EACjB,YAAY,EAAE,YAAY;IAgE5B,OAAO,CAAC,WAAW;CAKpB"}