@malloydata/db-trino 0.0.138-dev240401194552 → 0.0.138-dev240402170711

package/dist/trino_connection.d.ts

@@ -38,6 +38,8 @@ export declare class TrinoConnection implements Connection, PersistSQLResults {
     canStream(): this is StreamingConnection;
     get supportsNesting(): boolean;
     manifestTemporaryTable(_sqlCommand: string): Promise<string>;
+    convertRow(structDef: StructDef, row: unknown[]): {};
+    convertNest(structDef: StructDef, dataRows: unknown[][]): {};
     runSQL(sqlCommand: string, options?: RunSQLOptions, _rowIndex?: number): Promise<MalloyQueryData>;
     runSQLBlockAndFetchResultSchema(_sqlBlock: SQLBlock, _options?: RunSQLOptions): Promise<{
         data: MalloyQueryData;
@@ -57,6 +59,8 @@ export declare class TrinoConnection implements Connection, PersistSQLResults {
     }>;
     private structDefFromSqlBlock;
     private executeAndWait;
+    malloyTypeFromTrinoType(name: string, trinoType: string): FieldAtomicTypeDef | StructDef;
+    structDefFromSchema(rows: string[][], structDef: StructDef): void;
     private loadSchemaForSqlBlock;
     estimateQueryCost(_sqlCommand: string): Promise<QueryRunStats>;
     executeSQLRaw(_sqlCommand: string): Promise<QueryData>;

package/dist/trino_connection.js

@@ -42,8 +42,11 @@ class TrinoConnection {
             'integer': { type: 'number', numberType: 'integer' },
             'bigint': { type: 'number', numberType: 'integer' },
             'double': { type: 'number', numberType: 'float' },
+            'decimal': { type: 'number', numberType: 'float' },
             'string': { type: 'string' },
             'date': { type: 'date' },
+            'timestamp': { type: 'timestamp' },
+            'boolean': { type: 'boolean' },
             // TODO(figutierrez0): cleanup.
             /* 'INT64': {type: 'number', numberType: 'integer'},
             'FLOAT': {type: 'number', numberType: 'float'},
@@ -163,6 +166,25 @@ class TrinoConnection {
         throw maybeRewriteError(e);
       }
     }*/
+    convertRow(structDef, row) {
+        const col = {};
+        for (let i = 0; i < structDef.fields.length; i++) {
+            col[structDef.fields[i].name] = row[i];
+        }
+        return col;
+    }
+    convertNest(structDef, dataRows) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const ret = [];
+        if (structDef.structRelationship.type === 'nested' &&
+            !structDef.structRelationship.isArray) {
+            return this.convertRow(structDef, dataRows);
+        }
+        for (const row of dataRows) {
+            ret.push(this.convertRow(structDef, row));
+        }
+        return ret;
+    }
     async runSQL(sqlCommand, options = {}, 
     // TODO(figutierrez): Use.
     _rowIndex = 0) {
@@ -173,6 +195,10 @@ class TrinoConnection {
             // TODO: handle.
             throw new Error(`Failed to execute sql: ${sqlCommand}. \n Error: ${JSON.stringify(queryResult.value.error)}`);
         }
+        const malloyColumns = queryResult.value.columns.map(c => this.malloyTypeFromTrinoType(c.name, c.type));
+        // Debugging types
+        // const _x = queryResult.value.columns.map(c => console.log(c.type));
+        // console.log(JSON.stringify(malloyColumns, null, 2));
         let maxRows = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : 50;
         const malloyRows = [];
         while (queryResult !== null && maxRows--) {
@@ -181,9 +207,8 @@ class TrinoConnection {
                 const malloyRow = {};
                 for (let i = 0; i < queryResult.value.columns.length; i++) {
                     const column = queryResult.value.columns[i];
-                    // TODO: handle arrays etc.
-                    if (column.type === 'json') {
-                        malloyRow[column.name] = JSON.parse(row[i]);
+                    if (malloyColumns[i].type === 'struct') {
+                        malloyRow[column.name] = this.convertNest(malloyColumns[i], row[i]);
                     }
                     else {
                         malloyRow[column.name] = row[i];
@@ -315,8 +340,95 @@ class TrinoConnection {
         while (!(await result.next()).done)
             ;
     }
+    malloyTypeFromTrinoType(name, trinoType) {
+        var _a;
+        let malloyType;
+        // Arrays look like `array(type)`
+        const arrayMatch = trinoType.match(/^array\((.*)\)$/);
+        // Structs look like `row(name type, name type)`
+        const structMatch = trinoType.match(/^row\((.*)\)$/);
+        if (arrayMatch) {
+            const arrayType = arrayMatch[1];
+            const innerType = this.malloyTypeFromTrinoType(name, arrayType);
+            if (innerType.type === 'struct') {
+                malloyType = innerType;
+                malloyType.structRelationship = {
+                    type: 'nested',
+                    fieldName: name,
+                    isArray: true,
+                };
+            }
+            else {
+                malloyType = {
+                    type: 'struct',
+                    name,
+                    dialect: this.dialectName,
+                    structSource: { type: 'nested' },
+                    structRelationship: {
+                        type: 'nested',
+                        fieldName: name,
+                        isArray: true,
+                    },
+                    fields: [{ ...innerType, name: 'value' }],
+                };
+            }
+        }
+        else if (structMatch) {
+            // TODO: Trino doesn't quote or escape commas in field names,
+            // so some magic is going to need to be applied before we get here
+            // to avoid confusion if a field name contains a comma
+            const innerTypes = structMatch[1].split(/,\s+/);
+            malloyType = {
+                type: 'struct',
+                name,
+                dialect: this.dialectName,
+                structSource: { type: 'nested' },
+                structRelationship: {
+                    type: 'nested',
+                    fieldName: name,
+                    isArray: false,
+                },
+                fields: [],
+            };
+            for (let innerType of innerTypes) {
+                // TODO: Handle time zone type annotation, which is an
+                // exception to the types not containing spaces assumption
+                innerType = innerType.replace(/ with time zone$/, '');
+                const parts = innerType.match(/^(.*)\s(\S+)$/);
+                if (parts) {
+                    const innerName = parts[1];
+                    const innerTrinoType = parts[2];
+                    const innerMalloyType = this.malloyTypeFromTrinoType(innerName, innerTrinoType);
+                    malloyType.fields.push({ ...innerMalloyType, name: innerName });
+                }
+                else {
+                    malloyType.fields.push({
+                        name: 'unknown',
+                        type: 'unsupported',
+                        rawType: innerType.toLowerCase(),
+                    });
+                }
+            }
+        }
+        else {
+            malloyType = (_a = this.sqlToMalloyType(trinoType)) !== null && _a !== void 0 ? _a : {
+                type: 'unsupported',
+                rawType: trinoType.toLowerCase(),
+            };
+        }
+        return malloyType;
+    }
+    structDefFromSchema(rows, structDef) {
+        for (const row of rows) {
+            const name = row[0];
+            const type = row[1] || row[4];
+            const malloyType = this.malloyTypeFromTrinoType(name, type);
+            // console.log('>', row, '\n<', malloyType);
+            structDef.fields.push({ name, ...malloyType });
+        }
+    }
     async loadSchemaForSqlBlock(sqlBlock, structDef, element) {
-        var _a, _b;
+        var _a;
         try {
             const result = await this.trino.query(sqlBlock);
             const queryResult = await result.next();
@@ -325,15 +437,7 @@ class TrinoConnection {
                 throw new Error(`Failed to grab schema for ${element}: ${JSON.stringify(queryResult.value.error)}`);
             }
             const rows = (_a = queryResult.value.data) !== null && _a !== void 0 ? _a : [];
-            for (const row of rows) {
-                const fieldName = row[0];
-                const type = row[1];
-                const malloyType = (_b = this.sqlToMalloyType(type)) !== null && _b !== void 0 ? _b : {
-                    type: 'unsupported',
-                    rawType: type.toLowerCase(),
-                };
-                structDef.fields.push({ name: fieldName, ...malloyType });
-            }
+            this.structDefFromSchema(rows, structDef);
         }
         catch (e) {
             throw new Error(`Could not fetch schema for ${element} ${e}`);

package/dist/trino_connection.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/trino_connection.spec.js

@@ -0,0 +1,107 @@
+"use strict";
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const _1 = require(".");
+const ARRAY_SCHEMA = 'array(integer)';
+const STRUCT_SCHEMA = 'row(a double, b integer, c varchar(60))';
+const DEEP_SCHEMA = 'array(row(a double, b integer, c varchar(60)))';
+describe('Trino connection', () => {
+    describe('schema parser', () => {
+        it('parses arrays', () => {
+            const connection = new _1.TrinoConnection('trino', {}, _1.TrinoExecutor.getConnectionOptionsFromEnv());
+            expect(connection.malloyTypeFromTrinoType('test', ARRAY_SCHEMA)).toEqual({
+                'name': 'test',
+                'type': 'struct',
+                'dialect': 'trino',
+                'structRelationship': {
+                    'fieldName': 'test',
+                    'isArray': true,
+                    'type': 'nested',
+                },
+                'structSource': {
+                    'type': 'nested',
+                },
+                'fields': [
+                    {
+                        'name': 'value',
+                        'type': 'number',
+                        'numberType': 'integer',
+                    },
+                ],
+            });
+        });
+        it('parses structs', () => {
+            const connection = new _1.TrinoConnection('trino', {}, _1.TrinoExecutor.getConnectionOptionsFromEnv());
+            expect(connection.malloyTypeFromTrinoType('test', STRUCT_SCHEMA)).toEqual({
+                'name': 'test',
+                'type': 'struct',
+                'dialect': 'trino',
+                'structRelationship': {
+                    'fieldName': 'test',
+                    'isArray': false,
+                    'type': 'nested',
+                },
+                'structSource': {
+                    'type': 'nested',
+                },
+                'fields': [
+                    {
+                        'name': 'a',
+                        'type': 'number',
+                        'numberType': 'float',
+                    },
+                    {
+                        'name': 'b',
+                        'type': 'number',
+                        'numberType': 'integer',
+                    },
+                    {
+                        'name': 'c',
+                        'type': 'string',
+                    },
+                ],
+            });
+        });
+        it('parses arrays of structs', () => {
+            const connection = new _1.TrinoConnection('trino', {}, _1.TrinoExecutor.getConnectionOptionsFromEnv());
+            expect(connection.malloyTypeFromTrinoType('test', DEEP_SCHEMA)).toEqual({
+                'name': 'test',
+                'type': 'struct',
+                'dialect': 'trino',
+                'structRelationship': {
+                    'fieldName': 'test',
+                    'isArray': true,
+                    'type': 'nested',
+                },
+                'structSource': { 'type': 'nested' },
+                'fields': [
+                    { 'name': 'a', 'numberType': 'float', 'type': 'number' },
+                    { 'name': 'b', 'numberType': 'integer', 'type': 'number' },
+                    { 'name': 'c', 'type': 'string' },
+                ],
+            });
+        });
+    });
+});
+//# sourceMappingURL=trino_connection.spec.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-trino",
-  "version": "0.0.138-dev240401194552",
+  "version": "0.0.138-dev240402170711",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",