@cubejs-backend/athena-driver 0.29.25 → 0.29.28

package/CHANGELOG.md

@@ -3,6 +3,17 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.29.28](https://github.com/cube-js/cube.js/compare/v0.29.27...v0.29.28) (2022-02-10)
+
+
+### Bug Fixes
+
+* **@cubejs-backend/athena-driver:** Batching and export support ([#4039](https://github.com/cube-js/cube.js/issues/4039)) ([108f42a](https://github.com/cube-js/cube.js/commit/108f42afdd58ae0027b1b81730f7ca9e72ab9122))
+
+
+
+
+
 ## [0.29.25](https://github.com/cube-js/cube.js/compare/v0.29.24...v0.29.25) (2022-02-03)
 
 **Note:** Version bump only for package @cubejs-backend/athena-driver

package/dist/src/AthenaDriver.d.ts

@@ -0,0 +1,51 @@
+import { BaseDriver, DownloadTableCSVData, DriverInterface, QueryOptions, StreamOptions, StreamTableData } from '@cubejs-backend/query-orchestrator';
+import { AthenaClientConfig } from '@aws-sdk/client-athena/dist-types/AthenaClient';
+interface AthenaDriverOptions extends AthenaClientConfig {
+    readOnly?: boolean;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    workGroup?: string;
+    S3OutputLocation?: string;
+    pollTimeout?: number;
+    pollMaxInterval?: number;
+}
+export interface AthenaQueryId {
+    QueryExecutionId: string;
+}
+interface AthenaTable {
+    schema: string;
+    name: string;
+}
+interface AthenaColumn {
+    name: string;
+    type: string;
+    attributes: string[];
+}
+declare type AthenaSchema = Record<string, Record<string, AthenaColumn[]>>;
+export declare class AthenaDriver extends BaseDriver implements DriverInterface {
+    private config;
+    private athena;
+    constructor(config?: AthenaDriverOptions);
+    readOnly(): boolean;
+    isUnloadSupported(): Promise<boolean>;
+    testConnection(): Promise<void>;
+    query<R = unknown>(query: string, values: unknown[], options?: QueryOptions): Promise<R[]>;
+    stream(query: string, values: unknown[], options: StreamOptions): Promise<StreamTableData>;
+    loadPreAggregationIntoTable(preAggregationTableName: string, loadSql: string, params: any): Promise<any>;
+    unload(tableName: string): Promise<DownloadTableCSVData>;
+    tablesSchema(): Promise<AthenaSchema>;
+    protected startQuery(query: string, values: unknown[]): Promise<AthenaQueryId>;
+    protected checkStatus(qid: AthenaQueryId): Promise<boolean>;
+    protected waitForSuccess(qid: AthenaQueryId): Promise<void>;
+    protected lazyRowIterator<R extends unknown>(qid: AthenaQueryId, query: string): AsyncGenerator<R>;
+    protected viewsSchema(tablesSchema: AthenaSchema): Promise<AthenaSchema>;
+    protected getAllTables(): Promise<AthenaTable[]>;
+    protected getColumns(table: AthenaTable): Promise<AthenaSchema>;
+    protected mergeSchemas(arrSchemas: AthenaSchema[]): AthenaSchema;
+    static splitS3Path(path: string): {
+        bucket: string;
+        prefix: string;
+    };
+}
+export {};
+//# sourceMappingURL=AthenaDriver.d.ts.map

package/dist/src/AthenaDriver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AthenaDriver.d.ts","sourceRoot":"","sources":["../../src/AthenaDriver.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,UAAU,EACV,oBAAoB,EACpB,eAAe,EACf,YAAY,EAAE,aAAa,EAC3B,eAAe,EAChB,MAAM,oCAAoC,CAAC;AAG5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAGpF,UAAU,mBAAoB,SAAQ,kBAAkB;IACtD,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAID,MAAM,WAAW,aAAa;IAC5B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,UAAU,WAAW;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACb;AAED,UAAU,YAAY;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,EAAE,CAAA;CACrB;AAED,aAAK,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;AAMnE,qBAAa,YAAa,SAAQ,UAAW,YAAW,eAAe;IACrE,OAAO,CAAC,MAAM,CAAiC;IAE/C,OAAO,CAAC,MAAM,CAAS;gBAEJ,MAAM,GAAE,mBAAwB;IAmB5C,QAAQ,IAAI,OAAO;IAIb,iBAAiB;IAIjB,cAAc;IAMd,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAU1F,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAS1F,2BAA2B,CACtC,uBAAuB,EAAE,MAAM,EAC/B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,GAAG,GACV,OAAO,CAAC,GAAG,CAAC;IASF,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA6CxD,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC;cAOlC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC;cAmBpE,WAAW,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;cAejD,cAAc,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;cAehD,eAAe,CAAC,CAAC,SAAS,OAAO,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC;cAgCzF,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;cAc9D,YAAY,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;cAatC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAarE,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,YAAY,EAAE,GAAG,YAAY;WAelD,WAAW,CAAC,IAAI,EAAE,MAAM;;;;CAOvC"}

package/dist/src/AthenaDriver.js

@@ -0,0 +1,241 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AthenaDriver = void 0;
+const client_athena_1 = require("@aws-sdk/client-athena");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const stream = __importStar(require("stream"));
+const query_orchestrator_1 = require("@cubejs-backend/query-orchestrator");
+const shared_1 = require("@cubejs-backend/shared");
+const SqlString = __importStar(require("sqlstring"));
+const url_1 = require("url");
+function applyParams(query, params) {
+    return SqlString.format(query, params);
+}
+class AthenaDriver extends query_orchestrator_1.BaseDriver {
+    constructor(config = {}) {
+        super();
+        const accessKeyId = config.accessKeyId || process.env.CUBEJS_AWS_KEY;
+        const secretAccessKey = config.secretAccessKey || process.env.CUBEJS_AWS_SECRET;
+        this.config = {
+            credentials: accessKeyId && secretAccessKey ? { accessKeyId, secretAccessKey } : undefined,
+            region: config.region || process.env.CUBEJS_AWS_REGION,
+            S3OutputLocation: config.S3OutputLocation || process.env.CUBEJS_AWS_S3_OUTPUT_LOCATION,
+            workGroup: config.workGroup || process.env.CUBEJS_AWS_ATHENA_WORKGROUP || 'primary',
+            ...config,
+            pollTimeout: (config.pollTimeout || shared_1.getEnv('dbPollTimeout') || shared_1.getEnv('dbQueryTimeout')) * 1000,
+            pollMaxInterval: (config.pollMaxInterval || shared_1.getEnv('dbPollMaxInterval')) * 1000,
+        };
+        this.athena = new client_athena_1.Athena(this.config);
+    }
+    readOnly() {
+        return !!this.config.readOnly;
+    }
+    async isUnloadSupported() {
+        return this.config.S3OutputLocation !== undefined;
+    }
+    async testConnection() {
+        await this.athena.getWorkGroup({
+            WorkGroup: this.config.workGroup
+        });
+    }
+    async query(query, values, options) {
+        const qid = await this.startQuery(query, values);
+        await this.waitForSuccess(qid);
+        const rows = [];
+        for await (const row of this.lazyRowIterator(qid, query)) {
+            rows.push(row);
+        }
+        return rows;
+    }
+    async stream(query, values, options) {
+        const qid = await this.startQuery(query, values);
+        await this.waitForSuccess(qid);
+        const rowStream = stream.Readable.from(this.lazyRowIterator(qid, query), { highWaterMark: options.highWaterMark });
+        return {
+            rowStream
+        };
+    }
+    async loadPreAggregationIntoTable(preAggregationTableName, loadSql, params) {
+        if (this.config.S3OutputLocation === undefined) {
+            throw new Error('Unload is not configured');
+        }
+        const qid = await this.startQuery(loadSql, params);
+        await this.waitForSuccess(qid);
+    }
+    async unload(tableName) {
+        if (this.config.S3OutputLocation === undefined) {
+            throw new Error('Unload is not configured');
+        }
+        const path = `${this.config.S3OutputLocation}/${tableName}`;
+        const unloadSql = `
+      UNLOAD (SELECT * FROM ${tableName})
+      TO '${path}'
+      WITH (format = 'TEXTFILE', field_delimiter = ',', compression='GZIP')
+    `;
+        const qid = await this.startQuery(unloadSql, []);
+        await this.waitForSuccess(qid);
+        const client = new client_s3_1.S3({
+            credentials: this.config.credentials,
+            region: this.config.region,
+        });
+        const { bucket, prefix } = AthenaDriver.splitS3Path(path);
+        const list = await client.listObjectsV2({
+            Bucket: bucket,
+            // skip leading /
+            Prefix: prefix.slice(1),
+        });
+        if (list.Contents === undefined) {
+            throw new Error(`Unable to UNLOAD table ${path}`);
+        }
+        const csvFile = await Promise.all(list.Contents.map(async (file) => {
+            const command = new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: file.Key,
+            });
+            return s3_request_presigner_1.getSignedUrl(client, command, { expiresIn: 3600 });
+        }));
+        return {
+            csvFile,
+            csvNoHeader: true
+        };
+    }
+    async tablesSchema() {
+        const tablesSchema = await super.tablesSchema();
+        const viewsSchema = await this.viewsSchema(tablesSchema);
+        return this.mergeSchemas([tablesSchema, viewsSchema]);
+    }
+    async startQuery(query, values) {
+        const queryString = applyParams(query, (values || []).map(s => (typeof s === 'string' ? {
+            toSqlString: () => SqlString.escape(s).replace(/\\\\([_%])/g, '\\$1').replace(/\\'/g, '\'\'')
+        } : s)));
+        const { QueryExecutionId } = await this.athena.startQueryExecution({
+            QueryString: queryString,
+            WorkGroup: this.config.workGroup,
+            ResultConfiguration: {
+                OutputLocation: this.config.S3OutputLocation
+            }
+        });
+        return { QueryExecutionId: shared_1.checkNonNullable('StartQueryExecution', QueryExecutionId) };
+    }
+    async checkStatus(qid) {
+        var _a, _b, _c, _d;
+        const queryExecution = await this.athena.getQueryExecution(qid);
+        const status = (_b = (_a = queryExecution.QueryExecution) === null || _a === void 0 ? void 0 : _a.Status) === null || _b === void 0 ? void 0 : _b.State;
+        if (status === 'FAILED') {
+            throw new Error((_d = (_c = queryExecution.QueryExecution) === null || _c === void 0 ? void 0 : _c.Status) === null || _d === void 0 ? void 0 : _d.StateChangeReason);
+        }
+        if (status === 'CANCELLED') {
+            throw new Error('Query has been cancelled');
+        }
+        return status === 'SUCCEEDED';
+    }
+    async waitForSuccess(qid) {
+        const startedTime = Date.now();
+        for (let i = 0; Date.now() - startedTime <= this.config.pollTimeout; i++) {
+            if (await this.checkStatus(qid)) {
+                return;
+            }
+            await shared_1.pausePromise(Math.min(this.config.pollMaxInterval, 500 * i));
+        }
+        throw new Error(`Athena job timeout reached ${this.config.pollTimeout}ms`);
+    }
+    async *lazyRowIterator(qid, query) {
+        var _a, _b, _c, _d;
+        let isFirstBatch = true;
+        let columnInfo = [];
+        for (let results = await this.athena.getQueryResults(qid); results; results = results.NextToken
+            ? (await this.athena.getQueryResults({ ...qid, NextToken: results.NextToken }))
+            : undefined) {
+            let rows = (_b = (_a = results.ResultSet) === null || _a === void 0 ? void 0 : _a.Rows) !== null && _b !== void 0 ? _b : [];
+            if (isFirstBatch) {
+                isFirstBatch = false;
+                // Athena returns the columns names in first row, skip it.
+                rows = rows.slice(1);
+                columnInfo = /SHOW COLUMNS/.test(query) // Fix for getColumns method
+                    ? [{ Name: 'column' }]
+                    : shared_1.checkNonNullable('ColumnInfo', (_d = (_c = results.ResultSet) === null || _c === void 0 ? void 0 : _c.ResultSetMetadata) === null || _d === void 0 ? void 0 : _d.ColumnInfo)
+                        .map(info => ({ Name: shared_1.checkNonNullable('Name', info.Name) }));
+            }
+            for (const row of rows) {
+                const fields = {};
+                columnInfo
+                    .forEach((c, j) => {
+                    var _a;
+                    fields[c.Name] = (_a = row.Data) === null || _a === void 0 ? void 0 : _a[j].VarCharValue;
+                });
+                yield fields;
+            }
+        }
+    }
+    async viewsSchema(tablesSchema) {
+        const isView = (table) => !tablesSchema[table.schema]
+            || !tablesSchema[table.schema][table.name];
+        const allTables = await this.getAllTables();
+        const arrViewsSchema = await Promise.all(allTables
+            .filter(isView)
+            .map(table => this.getColumns(table)));
+        return this.mergeSchemas(arrViewsSchema);
+    }
+    async getAllTables() {
+        const data = await this.query(`
+        SELECT table_schema AS schema, table_name AS name
+        FROM information_schema.tables
+        WHERE tables.table_schema NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')
+      `, []);
+        return data;
+    }
+    async getColumns(table) {
+        const data = await this.query(`SHOW COLUMNS IN \`${table.schema}\`.\`${table.name}\``, []);
+        return {
+            [table.schema]: {
+                [table.name]: data.map(({ column }) => {
+                    const [name, type] = column.split('\t');
+                    return { name, type, attributes: [] };
+                })
+            }
+        };
+    }
+    mergeSchemas(arrSchemas) {
+        const result = {};
+        arrSchemas.forEach(schemas => {
+            Object.keys(schemas).forEach(schema => {
+                Object.keys(schemas[schema]).forEach((name) => {
+                    if (!result[schema])
+                        result[schema] = {};
+                    if (!result[schema][name])
+                        result[schema][name] = schemas[schema][name];
+                });
+            });
+        });
+        return result;
+    }
+    static splitS3Path(path) {
+        const url = new url_1.URL(path);
+        return {
+            bucket: url.host,
+            prefix: url.pathname
+        };
+    }
+}
+exports.AthenaDriver = AthenaDriver;
+//# sourceMappingURL=AthenaDriver.js.map

package/dist/src/AthenaDriver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AthenaDriver.js","sourceRoot":"","sources":["../../src/AthenaDriver.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,0DAA8E;AAC9E,kDAA0D;AAC1D,wEAA6D;AAC7D,+CAAiC;AACjC,2EAM4C;AAC5C,mDAA0F;AAC1F,qDAAuC;AAEvC,6BAA0B;AA+B1B,SAAS,WAAW,CAAC,KAAa,EAAE,MAAa;IAC/C,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED,MAAa,YAAa,SAAQ,+BAAU;IAK1C,YAAmB,SAA8B,EAAE;QACjD,KAAK,EAAE,CAAC;QAER,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QACrE,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAEhF,IAAI,CAAC,MAAM,GAAG;YACZ,WAAW,EAAE,WAAW,IAAI,eAAe,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,SAAS;YAC1F,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;YACtD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,6BAA6B;YACtF,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,SAAS;YACnF,GAAG,MAAM;YACT,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,IAAI,eAAM,CAAC,eAAe,CAAC,IAAI,eAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,IAAI;YAC/F,eAAe,EAAE,CAAC,MAAM,CAAC,eAAe,IAAI,eAAM,CAAC,mBAAmB,CAAC,CAAC,GAAG,IAAI;SAChF,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,sBAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAEM,QAAQ;QACb,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,gBAAgB,KAAK,SAAS,CAAC;IACpD,CAAC;IAEM,KAAK,CAAC,cAAc;QACzB,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;YAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAc,KAAa,EAAE,MAAiB,EAAE,OAAsB;QACtF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAQ,EAAE,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAI,GAAG,EAAE,KAAK,CAAC,EAAE;YAC3D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAChB;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAiB,EAAE,OAAsB;QAC1E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACjD,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;QACnH,OAAO;YACL,SAAS;SACV,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,2BAA2B,CACtC,uBAA+B,EAC/B,OAAe,EACf,MAAW;QAEX,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE;YAC9C,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,SAAiB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,KAAK,SAAS,EAAE;YAC9C,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,SAAS,EAAE,CAAC;QAE5D,MAAM,SAAS,GAAG;8BACQ,SAAS;YAC3B,IAAI;;KAEX,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAI,cAAE,CAAC;YACpB,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACpC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;SAC3B,CAAC,CAAC;QAEH,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;YACtC,MAAM,EAAE,MAAM;YACd,iBAAiB;YACjB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;SACxB,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;SACnD;QACD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC/B,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;gBACnC,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAC;YACH,OAAO,mCAAY,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC,CAAC,CACH,CAAC;QAEF,OAAO;YACL,OAAO;YACP,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,YAAY;QACvB,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAEzD,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;IACxD,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,MAAiB;QACzD,MAAM,WAAW,GAAG,WAAW,CAC7B,KAAK,EACL,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;YAC/C,WAAW,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC;SAC9F,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACR,CAAC;QAEF,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC;YACjE,WAAW,EAAE,WAAW;YACxB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,mBAAmB,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;aAC7C;SACF,CAAC,CAAC;QAEH,OAAO,EAAE,gBAAgB,EAAE,yBAAgB,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,EAAE,CAAC;IACzF,CAAC;IAES,KAAK,CAAC,WAAW,CAAC,GAAkB;;QAC5C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAEhE,MAAM,MAAM,eAAG,cAAc,CAAC,cAAc,0CAAE,MAAM,0CAAE,KAAK,CAAC;QAC5D,IAAI,MAAM,KAAK,QAAQ,EAAE;YACvB,MAAM,IAAI,KAAK,aAAC,cAAc,CAAC,cAAc,0CAAE,MAAM,0CAAE,iBAAiB,CAAC,CAAC;SAC3E;QAED,IAAI,MAAM,KAAK,WAAW,EAAE;YAC1B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,OAAO,MAAM,KAAK,WAAW,CAAC;IAChC,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,GAAkB;QAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE;YACxE,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE;gBAC/B,OAAO;aACR;YACD,MAAM,qBAAY,CAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,GAAG,GAAG,CAAC,CAAC,CAC/C,CAAC;SACH;QACD,MAAM,IAAI,KAAK,CACb,8BAA8B,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAC1D,CAAC;IACJ,CAAC;IAES,KAAK,CAAA,CAAE,eAAe,CAAoB,GAAkB,EAAE,KAAa;;QACnF,IAAI,YAAY,GAAG,IAAI,CAAC;QACxB,IAAI,UAAU,GAAuB,EAAE,CAAC;QACxC,KACE,IAAI,OAAO,GAA6C,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,EAC9F,OAAO,EACP,OAAO,GAAG,OAAO,CAAC,SAAS;YACzB,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,GAAG,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;YAC/E,CAAC,CAAC,SAAS,EACb;YACA,IAAI,IAAI,eAAG,OAAO,CAAC,SAAS,0CAAE,IAAI,mCAAI,EAAE,CAAC;YACzC,IAAI,YAAY,EAAE;gBAChB,YAAY,GAAG,KAAK,CAAC;gBACrB,0DAA0D;gBAC1D,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,4BAA4B;oBAClE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;oBACtB,CAAC,CAAC,yBAAgB,CAAC,YAAY,cAAE,OAAO,CAAC,SAAS,0CAAE,iBAAiB,0CAAE,UAAU,CAAC;yBAC/E,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,yBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;aACnE;YAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACtB,MAAM,MAAM,GAAwB,EAAE,CAAC;gBACvC,UAAU;qBACP,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;;oBAChB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAG,GAAG,CAAC,IAAI,0CAAG,CAAC,EAAE,YAAY,CAAC;gBAC9C,CAAC,CAAC,CAAC;gBACL,MAAM,MAAW,CAAC;aACnB;SACF;IACH,CAAC;IAES,KAAK,CAAC,WAAW,CAAC,YAA0B;QACpD,MAAM,MAAM,GAAG,CAAC,KAAkB,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC;eAC7D,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS;aACN,MAAM,CAAC,MAAM,CAAC;aACd,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CACxC,CAAC;QAEF,OAAO,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC;IAES,KAAK,CAAC,YAAY;QAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAC3B;;;;OAIC,EACD,EAAE,CACH,CAAC;QAEF,OAAO,IAAqB,CAAC;IAC/B,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,KAAkB;QAC3C,MAAM,IAAI,GAAyB,MAAM,IAAI,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,QAAQ,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAEjH,OAAO;YACL,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE;gBACd,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;oBACpC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACxC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;gBACxC,CAAC,CAAC;aACH;SACF,CAAC;IACJ,CAAC;IAES,YAAY,CAAC,UAA0B;QAC/C,MAAM,MAAM,GAAiB,EAAE,CAAC;QAEhC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC5C,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;wBAAE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;oBACzC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;wBAAE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC1E,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,MAAM,CAAC,WAAW,CAAC,IAAY;QACpC,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,OAAO;YACL,MAAM,EAAE,GAAG,CAAC,IAAI;YAChB,MAAM,EAAE,GAAG,CAAC,QAAQ;SACrB,CAAC;IACJ,CAAC;CACF;AAzQD,oCAyQC"}

package/dist/src/index.d.ts

@@ -0,0 +1,2 @@
+export * from './AthenaDriver';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,14 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./AthenaDriver"), exports);
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAA+B"}

package/index.js

@@ -0,0 +1,15 @@
+const fromExports = require('./dist/src');
+const { AthenaDriver } = require('./dist/src/AthenaDriver');
+
+/**
+ * After 5 years working with TypeScript, now I know
+ * that commonjs and nodejs require is not compatibility with using export default
+ */
+const toExport = AthenaDriver;
+
+// eslint-disable-next-line no-restricted-syntax
+for (const [key, module] of Object.entries(fromExports)) {
+  toExport[key] = module;
+}
+
+module.exports = toExport;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@cubejs-backend/athena-driver",
   "description": "Cube.js Athena database driver",
   "author": "Cube Dev, Inc.",
-  "version": "0.29.25",
+  "version": "0.29.28",
   "repository": {
     "type": "git",
     "url": "https://github.com/cube-js/cube.js.git",
@@ -12,22 +12,40 @@
     "node": "^12.0.0 || ^14.0.0 || >=16.0.0"
   },
   "scripts": {
-    "lint": "eslint src/index.d.ts"
+    "build": "rm -rf dist && npm run tsc",
+    "tsc": "tsc",
+    "watch": "tsc -w",
+    "test": "yarn integration",
+    "integration": "jest --verbose dist/test",
+    "lint": "eslint src/* --ext .ts",
+    "lint:fix": "eslint --fix src/* --ext .ts"
   },
-  "main": "src/AthenaDriver.ts",
-  "types": "src/index.d.ts",
+  "files": [
+    "dist/src",
+    "index.js"
+  ],
+  "main": "index.js",
+  "types": "dist/src/index.d.ts",
   "dependencies": {
     "@aws-sdk/client-athena": "^3.22.0",
-    "@cubejs-backend/query-orchestrator": "^0.29.25",
-    "@cubejs-backend/shared": "^0.29.23",
+    "@aws-sdk/client-s3": "^3.49.0",
+    "@aws-sdk/s3-request-presigner": "^3.49.0",
+    "@cubejs-backend/query-orchestrator": "^0.29.28",
+    "@cubejs-backend/shared": "^0.29.28",
     "sqlstring": "^2.3.1"
   },
   "devDependencies": {
-    "@cubejs-backend/linter": "^0.29.23"
+    "@cubejs-backend/linter": "^0.29.23",
+    "@cubejs-backend/testing": "^0.29.28",
+    "@types/ramda": "^0.27.40",
+    "typescript": "~4.1.5"
+  },
+  "publishConfig": {
+    "access": "public"
   },
   "eslintConfig": {
     "extends": "../cubejs-linter"
   },
   "license": "Apache-2.0",
-  "gitHead": "44ce6bb6bb9ec9ecfd98bbd155043efbf819a9fb"
+  "gitHead": "a258a9e046cbf0f4d60c3c0e694a8dc03ccad363"
 }

package/src/AthenaDriver.ts

@@ -1,183 +0,0 @@
-const AWS = require('@aws-sdk/client-athena');
-const { BaseDriver } = require('@cubejs-backend/query-orchestrator');
-const { getEnv, pausePromise } = require('@cubejs-backend/shared');
-const SqlString = require('sqlstring');
-
-const applyParams = (query, params) => SqlString.format(query, params);
-
-class AthenaDriver extends BaseDriver {
-  constructor(config = {}) {
-    super();
-
-    this.config = {
-      credentials: {
-        accessKeyId: config.accessKeyId || process.env.CUBEJS_AWS_KEY,
-        secretAccessKey: config.secretAccessKey || process.env.CUBEJS_AWS_SECRET,
-      },
-      region: process.env.CUBEJS_AWS_REGION,
-      S3OutputLocation: process.env.CUBEJS_AWS_S3_OUTPUT_LOCATION,
-      workGroup: process.env.CUBEJS_AWS_ATHENA_WORKGROUP || 'primary',
-      ...config,
-      pollTimeout: (config.pollTimeout || getEnv('dbPollTimeout') || getEnv('dbQueryTimeout')) * 1000,
-      pollMaxInterval: (config.pollMaxInterval || getEnv('dbPollMaxInterval')) * 1000,
-    };
-
-    this.athena = new AWS.Athena(this.config);
-  }
-
-  readOnly() {
-    return !!this.config.readOnly;
-  }
-
-  async testConnection() {
-    await this.athena.getWorkGroup({
-      WorkGroup: this.config.workGroup
-    });
-  }
-
-  async awaitForJobStatus(QueryExecutionId, query, options) {
-    const queryExecution = await this.athena.getQueryExecution({
-      QueryExecutionId
-    });
-
-    const status = queryExecution.QueryExecution.Status.State;
-    if (status === 'FAILED') {
-      throw new Error(queryExecution.QueryExecution.Status.StateChangeReason);
-    }
-
-    if (status === 'CANCELLED') {
-      throw new Error('Query has been cancelled');
-    }
-
-    if (
-      status === 'SUCCEEDED'
-    ) {
-      const allRows = [];
-      let columnInfo;
-
-      this.reportQueryUsage({
-        dataScannedInBytes: queryExecution.QueryExecution.Statistics.DataScannedInBytes
-      }, options);
-
-      for (
-        let results = await this.athena.getQueryResults({ QueryExecutionId });
-        results;
-        results = results.NextToken && (await this.athena.getQueryResults({
-          QueryExecutionId, NextToken: results.NextToken
-        }))
-      ) {
-        const [header, ...tableRows] = results.ResultSet.Rows;
-        allRows.push(...(allRows.length ? results.ResultSet.Rows : tableRows));
-        if (!columnInfo) {
-          columnInfo = /SHOW COLUMNS/.test(query) // Fix for getColumns method
-            ? [{ Name: 'column' }]
-            : results.ResultSet.ResultSetMetadata.ColumnInfo;
-        }
-      }
-
-      return allRows.map(r => columnInfo
-        .map((c, i) => ({ [c.Name]: r.Data[i].VarCharValue }))
-        .reduce((a, b) => ({ ...a, ...b }), {}));
-    }
-
-    return null;
-  }
-
-  async query(query, values, options) {
-    const queryString = applyParams(
-      query,
-      (values || []).map(s => (typeof s === 'string' ? {
-        toSqlString: () => SqlString.escape(s).replace(/\\\\([_%])/g, '\\$1').replace(/\\'/g, '\'\'')
-      } : s))
-    );
-
-    const { QueryExecutionId } = await this.athena.startQueryExecution({
-      QueryString: queryString,
-      WorkGroup: this.config.workGroup,
-      ResultConfiguration: {
-        OutputLocation: this.config.S3OutputLocation
-      }
-    });
-
-    const startedTime = Date.now();
-
-    for (let i = 0; Date.now() - startedTime <= this.config.pollTimeout; i++) {
-      const result = await this.awaitForJobStatus(QueryExecutionId, query, options);
-      if (result) {
-        return result;
-      }
-
-      await pausePromise(
-        Math.min(this.config.pollMaxInterval, 500 * i)
-      );
-    }
-
-    throw new Error(
-      `Athena job timeout reached ${this.config.pollTimeout}ms`
-    );
-  }
-
-  async tablesSchema() {
-    const tablesSchema = await super.tablesSchema();
-    const viewsSchema = await this.viewsSchema(tablesSchema);
-
-    return this.mergeSchemas([tablesSchema, viewsSchema]);
-  }
-
-  async viewsSchema(tablesSchema) {
-    // eslint-disable-next-line camelcase
-    const isView = ({ table_schema, table_name }) => !tablesSchema[table_schema]
-      || !tablesSchema[table_schema][table_name];
-
-    const allTables = await this.getAllTables();
-    const arrViewsSchema = await Promise.all(
-      allTables
-        .filter(isView)
-        .map(table => this.getColumns(table))
-    );
-
-    return this.mergeSchemas(arrViewsSchema);
-  }
-
-  async getAllTables() {
-    const data = await this.query(`
-      SELECT table_schema, table_name
-      FROM information_schema.tables
-      WHERE tables.table_schema NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')
-    `);
-
-    return data;
-  }
-
-  // eslint-disable-next-line camelcase
-  async getColumns({ table_schema, table_name } = {}) {
-    // eslint-disable-next-line camelcase
-    const data = await this.query(`SHOW COLUMNS IN \`${table_schema}\`.\`${table_name}\``);
-
-    return {
-      [table_schema]: {
-        [table_name]: data.map(({ column }) => {
-          const [name, type] = column.split('\t');
-          return { name, type, attributes: [] };
-        })
-      }
-    };
-  }
-
-  mergeSchemas(arrSchemas) {
-    const result = {};
-
-    arrSchemas.forEach(schemas => {
-      Object.keys(schemas).forEach(schema => {
-        Object.keys(schemas[schema]).forEach((name) => {
-          if (!result[schema]) result[schema] = {};
-          if (!result[schema][name]) result[schema][name] = schemas[schema][name];
-        });
-      });
-    });
-
-    return result;
-  }
-}
-
-module.exports = AthenaDriver;

package/src/index.d.ts

@@ -1,14 +0,0 @@
-import { BaseDriver } from '@cubejs-backend/query-orchestrator';
-import { AthenaClientConfig } from '@aws-sdk/client-athena';
-
-declare module '@cubejs-backend/athena-driver' {
-  interface AthenaDriverOptions extends AthenaClientConfig {
-    readOnly?: boolean,
-    pollTimeout?: number,
-    pollMaxInterval?: number,
-  }
-
-  export default class AthenaDriver extends BaseDriver {
-    public constructor(options?: AthenaDriverOptions);
-  }
-}