@malloydata/malloy 0.0.138-dev240328203849 → 0.0.138-dev240331175328

package/dist/dialect/dialect.d.ts

@@ -1,11 +1,19 @@
 import { DialectFragment, Expr, ExtractUnit, Sampling, StructDef, TimeFieldType, TimeValue, TimestampUnit, TypecastFragment, FieldAtomicTypeDef } from '../model/malloy_types';
 import { DialectFunctionOverloadDef } from './functions';
+type DialectFieldTypes = string | 'struct';
 interface DialectField {
-    type: string;
+    type: DialectFieldTypes;
     sqlExpression: string;
     rawName: string;
     sqlOutputName: string;
 }
+export interface DialectFieldTypeStruct extends DialectField {
+    type: 'struct';
+    nestedStruct: DialectFieldList;
+    isArray: boolean;
+}
+export declare function isDialectFieldStruct(d: DialectField): d is DialectFieldTypeStruct;
+export type DialectFieldList = DialectField[];
 /**
  * Data which dialect methods need in order to correctly generate SQL.
  * Initially this is just timezone related, but I made this an interface
@@ -20,7 +28,7 @@ export interface QueryInfo {
 export declare const dayIndex: number;
 export declare function inDays(units: string): boolean;
 export declare function qtz(qi: QueryInfo): string | undefined;
-export type DialectFieldList = DialectField[];
+export type OrderByClauseType = 'output_name' | 'ordinal';
 export declare abstract class Dialect {
     abstract name: string;
     abstract defaultNumberType: string;
@@ -42,6 +50,7 @@ export declare abstract class Dialect {
     supportsPipelinesInViews: boolean;
     supportsArraysInData: boolean;
     readsNestedData: boolean;
+    orderByClause: OrderByClauseType;
     abstract getGlobalFunctionDef(name: string): DialectFunctionOverloadDef[] | undefined;
     abstract quoteTablePath(tablePath: string): string;
     abstract sqlGroupSetTable(groupSetCount: number): string;

package/dist/dialect/dialect.js

@@ -22,8 +22,12 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Dialect = exports.qtz = exports.inDays = exports.dayIndex = void 0;
+exports.Dialect = exports.qtz = exports.inDays = exports.dayIndex = exports.isDialectFieldStruct = void 0;
 const malloy_types_1 = require("../model/malloy_types");
+function isDialectFieldStruct(d) {
+    return d.type === 'struct';
+}
+exports.isDialectFieldStruct = isDialectFieldStruct;
 const allUnits = [
     'microsecond',
     'millisecond',
@@ -64,6 +68,8 @@ class Dialect {
         this.supportsArraysInData = true;
         // can read some version of ga_sample
         this.readsNestedData = true;
+        // ORDER BY 1 DESC
+        this.orderByClause = 'ordinal';
     }
     sqlFinalStage(_lastStageName, _fields) {
         throw new Error('Dialect has no final Stage but called Anyway');

package/dist/dialect/trino/trino.d.ts

@@ -1,6 +1,6 @@
-import { Expr, ExtractUnit, Sampling, TimeValue, TimestampUnit, TypecastFragment, FieldAtomicTypeDef } from '../../model/malloy_types';
+import { Expr, ExtractUnit, Sampling, TimeValue, TimestampUnit, TypecastFragment, FieldAtomicTypeDef, DialectFragment } from '../../model/malloy_types';
 import { DialectFunctionOverloadDef } from '../functions';
-import { Dialect, DialectFieldList, QueryInfo } from '../dialect';
+import { Dialect, DialectFieldList, OrderByClauseType, QueryInfo } from '../dialect';
 export declare class TrinoDialect extends Dialect {
     name: string;
     defaultNumberType: string;
@@ -21,9 +21,12 @@ export declare class TrinoDialect extends Dialect {
     supportsSafeCast: boolean;
     supportsNesting: boolean;
     cantPartitionWindowFunctionsOnExpressions: boolean;
+    orderByClause: OrderByClauseType;
     quoteTablePath(tablePath: string): string;
     sqlGroupSetTable(groupSetCount: number): string;
+    dialectExpr(qi: QueryInfo, df: DialectFragment): Expr;
     sqlAnyValue(groupSet: number, fieldName: string): string;
+    buildTypeExpression(fieldList: DialectFieldList): string;
     sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined, limit: number | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;

package/dist/dialect/trino/trino.js

@@ -17,7 +17,7 @@
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,p
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
@@ -54,15 +54,19 @@ function qtz(qi) {
         return tz;
     }
 }
+const trinoTypeMap = {
+    'string': 'VARCHAR',
+    'number': 'DOUBLE',
+};
 class TrinoDialect extends dialect_1.Dialect {
     constructor() {
         super(...arguments);
         this.name = 'trino';
-        this.defaultNumberType = 'FLOAT64';
-        this.defaultDecimalType = 'NUMERIC';
+        this.defaultNumberType = 'DOUBLE';
+        this.defaultDecimalType = 'DECIMAL';
         this.udfPrefix = '__udf';
         this.hasFinalStage = false;
-        this.divisionIsInteger = false;
+        this.divisionIsInteger = true;
         this.supportsSumDistinctFunction = false;
         this.unnestWithNumbers = false;
         this.defaultSampling = { enable: false };
@@ -74,6 +78,7 @@ class TrinoDialect extends dialect_1.Dialect {
         this.supportsSafeCast = true;
         this.supportsNesting = true;
         this.cantPartitionWindowFunctionsOnExpressions = true;
+        this.orderByClause = 'output_name';
         // TODO(figutierrez): update.
         this.keywords = `
   ALL
@@ -174,24 +179,48 @@ class TrinoDialect extends dialect_1.Dialect {
     }
     quoteTablePath(tablePath) {
         // TODO: look into escaping.
-        return `${tablePath.replace(/malloytest/g, 'malloy_demo.faa')}`;
+        //return `${tablePath.replace(/malloytest/g, 'malloy_demo.malloytest')}`;
+        return tablePath;
     }
     sqlGroupSetTable(groupSetCount) {
         return `CROSS JOIN (SELECT row_number() OVER() -1  group_set FROM UNNEST(SEQUENCE(0,${groupSetCount})))`;
     }
+    dialectExpr(qi, df) {
+        switch (df.function) {
+            case 'div': {
+                return (0, malloy_types_1.mkExpr) `CAST(${df.numerator} AS DOUBLE)/${df.denominator}`;
+            }
+        }
+        return super.dialectExpr(qi, df);
+    }
     sqlAnyValue(groupSet, fieldName) {
         return `ANY_VALUE(CASE WHEN group_set=${groupSet} THEN ${fieldName} END)`;
     }
+    buildTypeExpression(fieldList) {
+        const fields = [];
+        for (const f of fieldList) {
+            if ((0, dialect_1.isDialectFieldStruct)(f)) {
+                let s = `ROW(${this.buildTypeExpression(f.nestedStruct)})`;
+                if (f.isArray) {
+                    s = `array(${s})`;
+                }
+                fields.push(s);
+            }
+            else {
+                fields.push(`${f.sqlOutputName} ${trinoTypeMap[f.type] || f.type}`);
+            }
+        }
+        return fields.join(', \n');
+    }
     // can array agg or any_value a struct...
     sqlAggregateTurtle(groupSet, fieldList, orderBy, limit) {
-        let tail = '';
+        const expressions = fieldList.map(f => f.sqlExpression).join(',\n ');
+        const definitions = this.buildTypeExpression(fieldList);
+        let ret = `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN CAST(ROW(${expressions}) AS ROW(${definitions})) END ${orderBy})`;
         if (limit !== undefined) {
-            tail += ` LIMIT ${limit}`;
+            ret = `SLICE(${ret}, 1, ${limit})`;
         }
-        const fields = fieldList
-            .map(f => `\n '${f.sqlOutputName}' VALUE ${f.sqlExpression}`)
-            .join(', ');
-        return `cast(COALESCE(SLICE(COALESCE(ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN JSON_PARSE(JSON_OBJECT(${fields})) END \n ${orderBy} -- ${tail}\n), ARRAY[]), 1, ${limit}), ARRAY[]) as json)`;
+        return ret;
     }
     sqlAnyValueTurtle(groupSet, fieldList) {
         const fields = fieldList
@@ -235,10 +264,9 @@ class TrinoDialect extends dialect_1.Dialect {
         }
     }
     sqlSumDistinctHashedKey(sqlDistinctKey) {
-        sqlDistinctKey = `CAST(${sqlDistinctKey} AS STRING)`;
-        const upperPart = `cast(cast(concat('0x', substr(to_hex(md5(${sqlDistinctKey})), 1, 15)) as int64) as numeric) * 4294967296`;
-        const lowerPart = `cast(cast(concat('0x', substr(to_hex(md5(${sqlDistinctKey})), 16, 8)) as int64) as numeric)`;
-        // See the comment below on `sql_sum_distinct` for why we multiply by this decimal
+        sqlDistinctKey = `CAST(${sqlDistinctKey} AS VARCHAR)`;
+        const upperPart = `cast(from_base(substr(to_hex(md5(to_utf8(${sqlDistinctKey}))), 1, 15),16) as DECIMAL) * DECIMAL '4294967296' `;
+        const lowerPart = `cast(from_base(substr(to_hex(md5(to_utf8(${sqlDistinctKey}))), 16, 8),16) as DECIMAL) `;
         const precisionShiftMultiplier = '0.000000001';
         return `(${upperPart} + ${lowerPart}) * ${precisionShiftMultiplier}`;
     }
@@ -425,12 +453,15 @@ ${(0, utils_1.indent)(sql)}
     malloyTypeToSQLType(malloyType) {
         if (malloyType.type === 'number') {
             if (malloyType.numberType === 'integer') {
-                return 'INT64';
+                return 'BIGINT';
             }
             else {
-                return 'FLOAT64';
+                return 'DOUBLE';
             }
         }
+        else if (malloyType.type === 'string') {
+            return 'VARCHAR';
+        }
         return malloyType.type;
     }
     sqlTypeToMalloyType(_sqlType) {
@@ -438,7 +469,7 @@ ${(0, utils_1.indent)(sql)}
         return undefined;
     }
     castToString(expression) {
-        return `CAST(${expression} as STRING)`;
+        return `CAST(${expression} as VARCHAR)`;
     }
     concat(...values) {
         return values.join(' || ');

package/dist/model/malloy_query.d.ts

@@ -268,6 +268,7 @@ declare class QueryQuery extends QueryField {
     generateDepthNFields(depth: number, resultSet: FieldInstanceResult, output: StageOutputContext, stageWriter: StageWriter): void;
     generateSQLDepthN(depth: number, stageWriter: StageWriter, stageName: string): string;
     genereateSQLCombineTurtles(stageWriter: StageWriter, stage0Name: string): string;
+    buildDialectFieldList(resultStruct: FieldInstanceResult): DialectFieldList;
     generateTurtleSQL(resultStruct: FieldInstanceResult, stageWriter: StageWriter, sqlFieldName: string, outputPipelinedSQL: OutputPipelinedSQL[]): string;
     generateTurtlePipelineSQL(fi: FieldInstanceResult, stageWriter: StageWriter, sourceSQLExpression: string): {
         structDef: StructDef;

package/dist/model/malloy_query.js

@@ -2101,14 +2101,25 @@ class QueryQuery extends QueryField {
                 // convert name to an index
                 const fi = resultStruct.getField(f.field);
                 if (fi && fi.fieldUsage.type === 'result') {
-                    o.push(`${fi.fieldUsage.resultIndex} ${f.dir || 'ASC'}`);
+                    if (this.parent.dialect.orderByClause === 'ordinal') {
+                        o.push(`${fi.fieldUsage.resultIndex} ${f.dir || 'ASC'}`);
+                    }
+                    else if (this.parent.dialect.orderByClause === 'output_name') {
+                        o.push(`${f.field}${f.dir || 'ASC'}`);
+                    }
                 }
                 else {
                     throw new Error(`Unknown field in ORDER BY ${f.field}`);
                 }
             }
             else {
-                o.push(`${f.field} ${f.dir || 'ASC'}`);
+                if (this.parent.dialect.orderByClause === 'ordinal') {
+                    o.push(`${f.field} ${f.dir || 'ASC'}`);
+                }
+                else if (this.parent.dialect.orderByClause === 'output_name') {
+                    const orderingField = resultStruct.getFieldByNumber(f.field);
+                    o.push(`${this.parent.dialect.sqlMaybeQuoteIdentifier(orderingField.name)} ${f.dir || 'ASC'}`);
+                }
             }
         }
         if (o.length > 0) {
@@ -2446,9 +2457,49 @@ class QueryQuery extends QueryField {
         this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter);
         return this.resultStage;
     }
+    // create a simplified version of the StructDef for dialects.
+    buildDialectFieldList(resultStruct) {
+        const dialectFieldList = [];
+        for (const [name, field] of resultStruct.allFields) {
+            const sqlName = this.parent.dialect.sqlMaybeQuoteIdentifier(name);
+            //
+            if (resultStruct.firstSegment.type === 'reduce' &&
+                field instanceof FieldInstanceResult) {
+                const d = {
+                    type: 'struct',
+                    sqlExpression: this.parent.dialect.sqlMaybeQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
+                    rawName: name,
+                    sqlOutputName: sqlName,
+                    isArray: field.getRepeatedResultType() === 'nested',
+                    nestedStruct: this.buildDialectFieldList(field),
+                };
+                dialectFieldList.push(d);
+            }
+            else if (resultStruct.firstSegment.type === 'reduce' &&
+                field instanceof FieldInstanceField &&
+                field.fieldUsage.type === 'result') {
+                dialectFieldList.push({
+                    type: field.f.fieldDef.type,
+                    sqlExpression: this.parent.dialect.sqlMaybeQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
+                    rawName: name,
+                    sqlOutputName: sqlName,
+                });
+            }
+            else if (resultStruct.firstSegment.type === 'project' &&
+                field instanceof FieldInstanceField &&
+                field.fieldUsage.type === 'result') {
+                dialectFieldList.push({
+                    type: field.type,
+                    sqlExpression: field.f.generateExpression(resultStruct),
+                    rawName: name,
+                    sqlOutputName: sqlName,
+                });
+            }
+        }
+        return dialectFieldList;
+    }
     generateTurtleSQL(resultStruct, stageWriter, sqlFieldName, outputPipelinedSQL) {
         // let fieldsSQL: string[] = [];
-        const dialectFieldList = [];
         let orderBy = '';
         const limit = (0, malloy_types_1.isRawSegment)(resultStruct.firstSegment)
             ? undefined
@@ -2480,38 +2531,7 @@ class QueryQuery extends QueryField {
         if (obSQL.length > 0) {
             orderBy = ' ' + this.parent.dialect.sqlOrderBy(obSQL);
         }
-        for (const [name, field] of resultStruct.allFields) {
-            const sqlName = this.parent.dialect.sqlMaybeQuoteIdentifier(name);
-            //
-            if (resultStruct.firstSegment.type === 'reduce' &&
-                (field instanceof FieldInstanceResult ||
-                    (field instanceof FieldInstanceField &&
-                        field.fieldUsage.type === 'result'))) {
-                // fieldsSQL.push(`${name}__${resultStruct.groupSet} as ${sqlName}`);
-                // outputFieldNames.push(name);
-                dialectFieldList.push({
-                    type: field instanceof FieldInstanceField
-                        ? field.f.fieldDef.type
-                        : 'struct',
-                    sqlExpression: this.parent.dialect.sqlMaybeQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
-                    rawName: name,
-                    sqlOutputName: sqlName,
-                });
-            }
-            else if (resultStruct.firstSegment.type === 'project' &&
-                field instanceof FieldInstanceField &&
-                field.fieldUsage.type === 'result') {
-                // fieldsSQL.push(
-                //   `${field.f.generateExpression(resultStruct)} as ${sqlName}`
-                // );
-                dialectFieldList.push({
-                    type: field.type,
-                    sqlExpression: field.f.generateExpression(resultStruct),
-                    rawName: name,
-                    sqlOutputName: sqlName,
-                });
-            }
-        }
+        const dialectFieldList = this.buildDialectFieldList(resultStruct);
         let resultType;
         let ret;
         if ((resultType = resultStruct.getRepeatedResultType()) !== 'nested') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.138-dev240328203849",
+  "version": "0.0.138-dev240331175328",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",