@malloydata/db-trino 0.0.133-dev240319171426 → 0.0.133

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { TrinoConnection } from './trino_connection';
+export { TrinoExecutor } from './trino_executor';

package/dist/{tmp_index.js → index.js}

@@ -22,4 +22,9 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=tmp_index.js.map
+exports.TrinoExecutor = exports.TrinoConnection = void 0;
+var trino_connection_1 = require("./trino_connection");
+Object.defineProperty(exports, "TrinoConnection", { enumerable: true, get: function () { return trino_connection_1.TrinoConnection; } });
+var trino_executor_1 = require("./trino_executor");
+Object.defineProperty(exports, "TrinoExecutor", { enumerable: true, get: function () { return trino_executor_1.TrinoExecutor; } });
+//# sourceMappingURL=index.js.map

package/dist/trino_connection.d.ts

@@ -0,0 +1,63 @@
+import { Connection, ConnectionConfig, FetchSchemaOptions, MalloyQueryData, FieldAtomicTypeDef, PersistSQLResults, PooledConnection, QueryData, QueryOptionsReader, QueryRunStats, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from '@malloydata/malloy';
+export interface TrinoManagerOptions {
+    credentials?: {
+        clientId: string;
+        clientSecret: string;
+        refreshToken: string | null;
+    };
+    projectId?: string | undefined;
+    userAgent: string;
+}
+export interface TrinoConnectionConfiguration {
+    server?: string;
+    catalog?: string;
+    schema?: string;
+    user?: string;
+    password?: string;
+}
+type TrinoConnectionOptions = ConnectionConfig;
+export declare class TrinoConnection implements Connection, PersistSQLResults {
+    trinoToMalloyTypes: {
+        [key: string]: FieldAtomicTypeDef;
+    };
+    private sqlToMalloyType;
+    readonly name: string;
+    private readonly dialect;
+    static DEFAULT_QUERY_OPTIONS: RunSQLOptions;
+    private schemaCache;
+    private sqlSchemaCache;
+    private queryOptions?;
+    private config;
+    private trino;
+    constructor(option: TrinoConnectionOptions, queryOptions?: QueryOptionsReader);
+    constructor(name: string, queryOptions?: QueryOptionsReader, config?: TrinoConnectionConfiguration);
+    get dialectName(): string;
+    private readQueryOptions;
+    isPool(): this is PooledConnection;
+    canPersist(): this is PersistSQLResults;
+    canStream(): this is StreamingConnection;
+    get supportsNesting(): boolean;
+    manifestTemporaryTable(_sqlCommand: string): Promise<string>;
+    runSQL(sqlCommand: string, options?: RunSQLOptions, _rowIndex?: number): Promise<MalloyQueryData>;
+    runSQLBlockAndFetchResultSchema(_sqlBlock: SQLBlock, _options?: RunSQLOptions): Promise<{
+        data: MalloyQueryData;
+        schema: StructDef;
+    }>;
+    fetchSchemaForTables(missing: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    private structDefFromTableSchema;
+    fetchSchemaForSQLBlock(_sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
+        structDef: StructDef;
+        error?: undefined;
+    } | {
+        error: string;
+        structDef?: undefined;
+    }>;
+    estimateQueryCost(_sqlCommand: string): Promise<QueryRunStats>;
+    executeSQLRaw(_sqlCommand: string): Promise<QueryData>;
+    test(): Promise<void>;
+    close(): Promise<void>;
+}
+export {};

package/dist/trino_connection.js

@@ -0,0 +1,335 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TrinoConnection = void 0;
+const malloy_1 = require("@malloydata/malloy");
+const trino_client_1 = require("trino-client");
+// manage access to BQ, control costs, enforce global data/API limits
+class TrinoConnection {
+    sqlToMalloyType(sqlType) {
+        var _a, _b;
+        const baseSqlType = (_b = (_a = sqlType.match(/^(\w+)/)) === null || _a === void 0 ? void 0 : _a.at(0)) !== null && _b !== void 0 ? _b : sqlType;
+        if (this.trinoToMalloyTypes[baseSqlType]) {
+            return this.trinoToMalloyTypes[baseSqlType];
+        }
+        return undefined;
+    }
+    constructor(arg, queryOptions, config = {}) {
+        this.trinoToMalloyTypes = {
+            'varchar': { type: 'string' },
+            'integer': { type: 'number', numberType: 'integer' },
+            'bigint': { type: 'number', numberType: 'integer' },
+            'double': { type: 'number', numberType: 'float' },
+            'string': { type: 'string' },
+            'date': { type: 'date' },
+            // TODO(figutierrez0): cleanup.
+            /* 'INT64': {type: 'number', numberType: 'integer'},
+            'FLOAT': {type: 'number', numberType: 'float'},
+            'FLOAT64': {type: 'number', numberType: 'float'},
+            'NUMERIC': {type: 'number', numberType: 'float'},
+            'BIGNUMERIC': {type: 'number', numberType: 'float'},
+            'TIMESTAMP': {type: 'timestamp'},
+            'BOOLEAN': {type: 'boolean'},
+            'BOOL': {type: 'boolean'},
+            'JSON': {type: 'json'},*/
+            // TODO (https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#tablefieldschema):
+            // BYTES
+            // DATETIME
+            // TIME
+            // GEOGRAPHY
+        };
+        this.dialect = new malloy_1.StandardSQLDialect();
+        this.schemaCache = new Map();
+        this.sqlSchemaCache = new Map();
+        this.name = 'trino';
+        /* if (typeof arg === 'string') {
+          this.name = arg;
+        } else {
+          const {name, client_email, private_key, ...args} = arg;
+          this.name = name;
+          config = args;
+          if (client_email || private_key) {
+            config.credentials = {
+              client_email,
+              private_key,
+            };
+          }
+        }*/
+        // TODO: check user is set.
+        this.trino = trino_client_1.Trino.create({
+            server: config.server,
+            catalog: config.catalog,
+            schema: config.schema,
+            auth: new trino_client_1.BasicAuth(config.user, config.password),
+        });
+        this.queryOptions = queryOptions;
+        this.config = config;
+    }
+    get dialectName() {
+        return 'trino';
+    }
+    readQueryOptions() {
+        const options = TrinoConnection.DEFAULT_QUERY_OPTIONS;
+        if (this.queryOptions) {
+            if (this.queryOptions instanceof Function) {
+                return { ...options, ...this.queryOptions() };
+            }
+            else {
+                return { ...options, ...this.queryOptions };
+            }
+        }
+        else {
+            return options;
+        }
+    }
+    isPool() {
+        return false;
+    }
+    canPersist() {
+        return true;
+    }
+    canStream() {
+        return false;
+    }
+    get supportsNesting() {
+        return true;
+    }
+    async manifestTemporaryTable(_sqlCommand) {
+        throw new Error('not implemented 1');
+    }
+    /*  private async _runSQL(
+      sqlCommand: string,
+      {rowLimit, abortSignal}: RunSQLOptions = {},
+      rowIndex = 0
+    ): Promise<{
+      data: MalloyQueryData;
+      schema: Trino.ITableFieldSchema | undefined;
+    }> {
+      const defaultOptions = this.readQueryOptions();
+      const pageSize = rowLimit ?? defaultOptions.rowLimit;
+  
+      try {
+        const queryResultsOptions: QueryResultsOptions = {
+          maxResults: pageSize,
+          startIndex: rowIndex.toString(),
+        };
+  
+        const jobResult = await this.createTrinoJobAndGetResults(
+          sqlCommand,
+          undefined,
+          queryResultsOptions,
+          abortSignal
+        );
+  
+        const totalRows = +(jobResult[2]?.totalRows
+          ? jobResult[2].totalRows
+          : '0');
+  
+        // TODO even though we have 10 minute timeout limit, we still should confirm that resulting metadata has "jobComplete: true"
+        const queryCostBytes = jobResult[2]?.totalBytesProcessed;
+        const data: MalloyQueryData = {
+          rows: jobResult[0],
+          totalRows,
+          runStats: {
+            queryCostBytes: queryCostBytes ? +queryCostBytes : undefined,
+          },
+        };
+        const schema = jobResult[2]?.schema;
+  
+        return {data, schema};
+      } catch (e) {
+        throw maybeRewriteError(e);
+      }
+    }*/
+    async runSQL(sqlCommand, options = {}, 
+    // TODO(figutierrez): Use.
+    _rowIndex = 0) {
+        var _a, _b;
+        //sqlCommand = sqlCommand.replace(new RegExp('`', 'g'), '');
+        // TODO: default row limit.
+        sqlCommand = `SELECT * FROM (${sqlCommand}) limit ${(_a = options.rowLimit) !== null && _a !== void 0 ? _a : 50}`;
+        // TODO: fill in with options.
+        const result = await this.trino.query(sqlCommand);
+        let queryResult = await result.next();
+        if (queryResult.value.error) {
+            // TODO: handle.
+            throw new Error(`Failed to execute sql: ${sqlCommand}. \n Error: ${JSON.stringify(queryResult.value.error)}`);
+        }
+        const malloyRows = [];
+        while (queryResult !== null) {
+            const rows = (_b = queryResult.value.data) !== null && _b !== void 0 ? _b : [];
+            for (const row of rows) {
+                const malloyRow = {};
+                for (let i = 0; i < queryResult.value.columns.length; i++) {
+                    const column = queryResult.value.columns[i];
+                    // TODO: handle arrays etc.
+                    if (column.type === 'json') {
+                        malloyRow[column.name] = JSON.parse(row[i]);
+                    }
+                    else {
+                        malloyRow[column.name] = row[i];
+                    }
+                }
+                malloyRows.push(malloyRow);
+            }
+            if (!queryResult.done) {
+                queryResult = await result.next();
+            }
+            else {
+                break;
+            }
+        }
+        // TODO(figutierrez): Remove.
+        // eslint-disable-next-line no-console
+        console.log(`ROWS: ${JSON.stringify(malloyRows)} ${malloyRows.length}`);
+        // TODO: handle totalrows.
+        return { rows: malloyRows, totalRows: malloyRows.length };
+    }
+    async runSQLBlockAndFetchResultSchema(_sqlBlock, _options) {
+        /*const {data, schema: schemaRaw} = await this._runSQL(
+          sqlBlock.selectStr,
+          options
+        );
+    
+        // TODO need to probably surface the cause of the schema not present error
+        if (schemaRaw === undefined) {
+          throw new Error('Schema not present');
+        }
+    
+        const schema = this.structDefFromSQLSchema(sqlBlock, schemaRaw);
+        return {data, schema};*/
+        throw new Error('Not implemented 3');
+    }
+    async fetchSchemaForTables(missing, { refreshTimestamp }) {
+        const schemas = {};
+        const errors = {};
+        for (const tableKey in missing) {
+            let inCache = this.schemaCache.get(tableKey);
+            const tablePath = missing[tableKey].replace(/malloytest/g, 'malloy_demo.faa');
+            if (!inCache ||
+                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
+                const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
+                try {
+                    const schema = await this.structDefFromTableSchema(tableKey, 
+                    // TODO: remove.
+                    tablePath);
+                    inCache = {
+                        schema,
+                        timestamp,
+                    };
+                    this.schemaCache.set(tableKey, inCache);
+                }
+                catch (error) {
+                    inCache = { error: error.message, timestamp };
+                }
+            }
+            if (inCache.schema !== undefined) {
+                schemas[tableKey] = inCache.schema;
+            }
+            else {
+                errors[tableKey] = inCache.error || 'Unknown schema fetch error';
+            }
+        }
+        return { schemas: schemas, errors: errors };
+    }
+    async structDefFromTableSchema(tableKey, tablePath) {
+        var _a, _b;
+        const structDef = {
+            type: 'struct',
+            name: tableKey,
+            dialect: this.dialectName,
+            structSource: {
+                type: 'table',
+                tablePath,
+            },
+            structRelationship: {
+                type: 'basetable',
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        try {
+            const result = await this.trino.query(`DESCRIBE ${tablePath}`);
+            const queryResult = await result.next();
+            if (queryResult.value.error) {
+                // TODO: handle.
+                throw new Error(`Failed to grab schema for table ${tablePath}: ${JSON.stringify(queryResult.value.error)}`);
+            }
+            const rows = (_a = queryResult.value.data) !== null && _a !== void 0 ? _a : [];
+            for (const row of rows) {
+                const fieldName = row[0];
+                const type = row[1];
+                const malloyType = (_b = this.sqlToMalloyType(type)) !== null && _b !== void 0 ? _b : {
+                    type: 'unsupported',
+                    rawType: type.toLowerCase(),
+                };
+                structDef.fields.push({ name: fieldName, ...malloyType });
+            }
+        }
+        catch (e) {
+            throw new Error(`Could not fetch schema for table ${tablePath} ${e}`);
+        }
+        // TODO: handle repeated etc.
+        return structDef;
+    }
+    async fetchSchemaForSQLBlock(_sqlRef, 
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    { refreshTimestamp }) {
+        throw new Error('Not implemented 5');
+    }
+    /*  public async downloadMalloyQuery(
+      sqlCommand: string
+    ): Promise<ResourceStream<RowMetadata>> {
+      const job = await this.createTrinoJob({
+        query: sqlCommand,
+      });
+  
+      return job.getQueryResultsStream();
+    }*/
+    async estimateQueryCost(_sqlCommand) {
+        /*const dryRunResults = await this.dryRunSQLQuery(sqlCommand);
+        return {
+          queryCostBytes: Number(
+            dryRunResults.metadata.statistics.totalBytesProcessed
+          ),
+        };*/
+        throw new Error('Not impld 6');
+    }
+    async executeSQLRaw(_sqlCommand) {
+        /*const result = await this.createTrinoJobAndGetResults(sqlCommand);
+        return result[0];*/
+        throw new Error('Not implemented 7');
+    }
+    async test() {
+        // await this.dryRunSQLQuery('SELECT 1');
+    }
+    async close() {
+        return;
+    }
+}
+exports.TrinoConnection = TrinoConnection;
+TrinoConnection.DEFAULT_QUERY_OPTIONS = {
+    rowLimit: 10,
+};
+//# sourceMappingURL=trino_connection.js.map

package/dist/trino_executor.d.ts

@@ -0,0 +1,4 @@
+import { TrinoConnectionConfiguration } from './trino_connection';
+export declare class TrinoExecutor {
+    static getConnectionOptionsFromEnv(): TrinoConnectionConfiguration | undefined;
+}

package/dist/trino_executor.js

@@ -0,0 +1,53 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TrinoExecutor = void 0;
+class TrinoExecutor {
+    static getConnectionOptionsFromEnv() {
+        const server = process.env['TRINO_SERVER'];
+        if (server) {
+            const user = process.env['TRINO_USER'];
+            if (!user) {
+                throw Error('Trino server specified but no user was provided. Set TRINO_USER and TRINO_PASSWORD environment variables');
+            }
+            const password = process.env['TRINO_PASSWORD'];
+            // TODO(figutierrez): We may not need to support these.
+            const catalog = process.env['TRINO_CATALOG'];
+            const schema = process.env['TRINO_SCHEMA'];
+            return {
+                server,
+                user,
+                password,
+                catalog,
+                schema,
+            };
+        }
+        else {
+            throw new Error('Tried to get Trino connection from Env, no TRINO_SERVER specified.');
+        }
+        return undefined;
+    }
+}
+exports.TrinoExecutor = TrinoExecutor;
+//# sourceMappingURL=trino_executor.js.map

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@malloydata/db-trino",
-  "version": "0.0.133-dev240319171426",
+  "version": "0.0.133",
   "license": "MIT",
-  "main": "dist/tmp_index.js",
-  "types": "dist/tmp_index.d.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "homepage": "https://github.com/malloydata/malloy#readme",
   "repository": {
     "type": "git",

package/dist/tmp_index.d.ts

@@ -1 +0,0 @@
-export {};