@malloydata/malloy 0.0.350 → 0.0.352

package/dist/dialect/databricks/function_overrides.js

@@ -0,0 +1,68 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DATABRICKS_MALLOY_STANDARD_OVERLOADS = void 0;
+// Template strings use ${...values} for Malloy parameter interpolation.
+// The ELSE branch uses a JS template literal to inject `name`, so the
+// Malloy parameter reference must be escaped as \${...values}.
+function greatestOrLeastSQL(name) {
+    return ('CASE' +
+        ' WHEN SIZE(FILTER(ARRAY(${...values}), x -> x IS NULL)) > 0' +
+        ' THEN NULL' +
+        ` ELSE ${name}(\${...values})` +
+        ' END');
+}
+exports.DATABRICKS_MALLOY_STANDARD_OVERLOADS = {
+    // Databricks REGEXP_EXTRACT defaults to group index 1 (first capture group),
+    // but Malloy expects the full match (group 0). Explicitly pass idx=0.
+    regexp_extract: { sql: 'REGEXP_EXTRACT(${value}, ${pattern}, 0)' },
+    replace: {
+        regular_expression: {
+            sql: 'REGEXP_REPLACE(${value}, ${pattern}, ${replacement})',
+        },
+    },
+    trunc: {
+        to_integer: {
+            sql: 'CAST(${value} AS BIGINT)',
+        },
+        to_precision: {
+            sql: '(ABS(FLOOR(${value} * POW(10,${precision}))/POW(10,${precision}))*IF(${value} < 0, -1, 1))',
+        },
+    },
+    log: { sql: 'LOG(${base},${value})' },
+    div: { sql: 'FLOOR(${dividend} / ${divisor})' },
+    strpos: { sql: 'LOCATE(${search_string},${test_string})' },
+    starts_with: { sql: 'COALESCE(STARTSWITH(${value},${prefix}), false)' },
+    ends_with: { sql: 'COALESCE(ENDSWITH(${value},${suffix}), false)' },
+    trim: {
+        characters: {
+            sql: 'TRIM(BOTH ${trim_characters} FROM ${value})',
+        },
+    },
+    ltrim: {
+        characters: {
+            sql: 'TRIM(LEADING ${trim_characters} FROM ${value})',
+        },
+    },
+    rtrim: {
+        characters: {
+            sql: 'TRIM(TRAILING ${trim_characters} FROM ${value})',
+        },
+    },
+    byte_length: { sql: 'OCTET_LENGTH(${value})' },
+    chr: { sql: 'CHR(${value})' },
+    // Databricks has no IS_INF/IS_NAN functions; compare to special float values
+    is_inf: {
+        sql: "COALESCE(${value} = DOUBLE('infinity') OR ${value} = DOUBLE('-infinity'), false)",
+    },
+    is_nan: { sql: "COALESCE(${value} = DOUBLE('NaN'), false)" },
+    // Databricks ASCII() returns the Unicode codepoint of the first character
+    unicode: { function: 'ASCII' },
+    // Databricks GREATEST/LEAST skip nulls; Malloy expects null propagation
+    greatest: { sql: greatestOrLeastSQL('GREATEST') },
+    least: { sql: greatestOrLeastSQL('LEAST') },
+};
+//# sourceMappingURL=function_overrides.js.map

package/dist/dialect/databricks/index.d.ts

@@ -0,0 +1 @@
+export * from './databricks';

package/dist/dialect/databricks/index.js

@@ -0,0 +1,22 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./databricks"), exports);
+//# sourceMappingURL=index.js.map

package/dist/dialect/dialect.d.ts

@@ -7,6 +7,26 @@ interface DialectField {
     sqlOutputName: string;
 }
 export type DialectFieldList = DialectField[];
+/**
+ * Order-by entry with field references resolved to SQL expressions
+ * and direction defaulted. Used by sqlAggregateTurtle so each dialect
+ * can format ordering in its own syntax.
+ */
+export interface CompiledOrderBy {
+    /** SQL expression for the ordering column (e.g. the group_set-suffixed CTE column name) */
+    field: string;
+    /** The struct output field name, for dialects that sort via struct field reference */
+    structField: string;
+    dir: 'asc' | 'desc';
+}
+/**
+ * A named expression for the lateral join bag. The expression will be
+ * available as `__lateral_join_bag.name` in the query.
+ */
+export interface LateralJoinExpression {
+    sql: string;
+    name: string;
+}
 export declare const MIN_INT32 = -2147483648;
 export declare const MAX_INT32 = 2147483647;
 export declare const MIN_INT64: bigint;
@@ -64,6 +84,7 @@ export declare abstract class Dialect {
     abstract supportsNesting: boolean;
     abstract experimental: boolean;
     cantPartitionWindowFunctionsOnExpressions: boolean;
+    sqlLateralJoinBag(_expressions: LateralJoinExpression[]): string;
     supportsPipelinesInViews: boolean;
     supportsArraysInData: boolean;
     hasTimestamptz: boolean;
@@ -78,6 +99,7 @@ export declare abstract class Dialect {
     hasModOperator: boolean;
     supportsLeftJoinUnnest: boolean;
     requiresExplicitUnnestOrdering: boolean;
+    hasLateralColumnAliasInSelect: boolean;
     supportsCountApprox: boolean;
     supportsHyperLogLog: boolean;
     supportsFullJoin: boolean;
@@ -113,7 +135,8 @@ export declare abstract class Dialect {
     abstract quoteTablePath(tablePath: string): string;
     abstract sqlGroupSetTable(groupSetCount: number): string;
     abstract sqlAnyValue(groupSet: number, fieldName: string): string;
-    abstract sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    abstract sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
+    sqlTurtleOrderByClause(orderBy: CompiledOrderBy[]): string;
     abstract sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     abstract sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     abstract sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/dialect.js

@@ -72,7 +72,10 @@ class Dialect {
         // -- we should add flags with default values from now on so as to not break
         // dialects outside our repository
         //
-        // StandardSQL dialects can't partition on expression in window functions
+        // StandardSQL dialects can't partition on expression in window functions.
+        // When true, dimension expressions used in PARTITION BY are moved to a
+        // lateral join bag so the PARTITION BY can reference a column name instead
+        // of a raw expression. See sqlLateralJoinBag for dialect-specific syntax.
         this.cantPartitionWindowFunctionsOnExpressions = false;
         // Snowflake can't yet support pipelines in nested views.
         this.supportsPipelinesInViews = true;
@@ -104,6 +107,16 @@ class Dialect {
         // UNNEST in LATERAL JOINs doesn't guarantee array element order.
         // When true, compiler adds ORDER BY on array ordinality columns (__row_id)
         this.requiresExplicitUnnestOrdering = false;
+        // In most SQL dialects, column aliases defined in a SELECT clause are not
+        // visible to other expressions in the same SELECT. However, some dialects
+        // (e.g. Databricks/Spark) support "lateral column aliases", where an alias
+        // can be referenced by later expressions in the same SELECT. This causes
+        // problems when the compiler remaps the `group_set` column in a combine-
+        // turtles stage: the alias shadows the input column, so CASE WHEN
+        // group_set=N checks in aggregate expressions see the remapped value
+        // instead of the original. When true, the compiler splits the group_set
+        // remapping into a separate CTE to avoid shadowing.
+        this.hasLateralColumnAliasInSelect = false;
         this.supportsCountApprox = false;
         this.supportsHyperLogLog = false;
         // MYSQL doesn't have full join, maybe others.
@@ -126,6 +139,16 @@ class Dialect {
             { min: exports.MIN_INT64, max: exports.MAX_INT64, numberType: 'bigint' },
         ];
     }
+    // Generate the lateral join bag clause for window function partitioning.
+    // The expressions are dimension fields that need to be referenced by name
+    // in PARTITION BY clauses. Must be overridden by any dialect that sets
+    // cantPartitionWindowFunctionsOnExpressions = true.
+    sqlLateralJoinBag(_expressions) {
+        if (this.cantPartitionWindowFunctionsOnExpressions) {
+            throw new Error(`Dialect '${this.name}' sets cantPartitionWindowFunctionsOnExpressions but does not implement sqlLateralJoinBag`);
+        }
+        throw new Error('Internal error: sqlLateralJoinBag called but cantPartitionWindowFunctionsOnExpressions is false');
+    }
     /**
      * Determine the Malloy number type for a numeric literal.
      */
@@ -195,6 +218,13 @@ class Dialect {
             },
         };
     }
+    // Format a CompiledOrderBy[] into an ORDER BY clause string for use
+    // inside an aggregate turtle expression. Dialects which support ORDER BY
+    // inside aggregate functions can call this helper from sqlAggregateTurtle.
+    sqlTurtleOrderByClause(orderBy) {
+        const terms = orderBy.map(o => ` ${o.field} ${o.dir.toUpperCase()}`);
+        return ' ' + this.sqlOrderBy(terms, 'turtle');
+    }
     sqlFinalStage(_lastStageName, _fields) {
         throw new Error('Dialect has no final Stage but called Anyway');
     }

package/dist/dialect/dialect_map.js

@@ -31,6 +31,7 @@ const snowflake_1 = require("./snowflake");
 const standardsql_1 = require("./standardsql");
 const trino_1 = require("./trino");
 const mysql_1 = require("./mysql");
+const databricks_1 = require("./databricks");
 const dialectMap = new Map();
 function getDialect(name) {
     const d = dialectMap.get(name);
@@ -52,4 +53,5 @@ registerDialect(new snowflake_1.SnowflakeDialect());
 registerDialect(new trino_1.TrinoDialect());
 registerDialect(new trino_1.PrestoDialect());
 registerDialect(new mysql_1.MySQLDialect());
+registerDialect(new databricks_1.DatabricksDialect());
 //# sourceMappingURL=dialect_map.js.map

package/dist/dialect/duckdb/duckdb.d.ts

@@ -1,6 +1,6 @@
 import type { Sampling, AtomicTypeDef, RegexMatchExpr, MeasureTimeExpr, BasicAtomicTypeDef, RecordLiteralNode, OrderBy, TimestampUnit } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import type { DialectFieldList, FieldReferenceType, IntegerTypeMapping } from '../dialect';
+import type { CompiledOrderBy, DialectFieldList, FieldReferenceType, IntegerTypeMapping } from '../dialect';
 import { PostgresBase } from '../pg_impl';
 export declare class DuckDBDialect extends PostgresBase {
     name: string;
@@ -30,7 +30,7 @@ export declare class DuckDBDialect extends PostgresBase {
     sqlAnyValue(groupSet: number, fieldName: string): string;
     sqlLiteralNumber(literal: string): string;
     mapFields(fieldList: DialectFieldList): string;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/duckdb/duckdb.js

@@ -109,7 +109,8 @@ class DuckDBDialect extends pg_impl_1.PostgresBase {
         const fields = fieldList
             .map(f => `\n  ${f.sqlOutputName}: ${f.sqlExpression}`)
             .join(', ');
-        return `COALESCE(LIST({${fields}} ${orderBy}) FILTER (WHERE group_set=${groupSet}),[])`;
+        const orderByClause = orderBy ? this.sqlTurtleOrderByClause(orderBy) : '';
+        return `COALESCE(LIST({${fields}} ${orderByClause}) FILTER (WHERE group_set=${groupSet}),[])`;
     }
     sqlAnyValueTurtle(groupSet, fieldList) {
         const fields = fieldList

package/dist/dialect/index.d.ts

@@ -1,13 +1,14 @@
 export type { DialectFunctionOverloadDef, DefinitionBlueprint, DefinitionBlueprintMap, OverloadedDefinitionBlueprint, } from './functions/util';
 export { arg, anyExprType, makeParam, overload, minScalar, minAggregate, maxScalar, spread, param, variadicParam, literal, sql, } from './functions/util';
 export { Dialect, qtz } from './dialect';
-export type { DialectFieldList, QueryInfo, FieldReferenceType } from './dialect';
+export type { DialectFieldList, CompiledOrderBy, LateralJoinExpression, QueryInfo, FieldReferenceType, } from './dialect';
 export { StandardSQLDialect } from './standardsql';
 export { PostgresDialect } from './postgres';
 export { DuckDBDialect } from './duckdb';
 export { SnowflakeDialect } from './snowflake';
 export { TrinoDialect } from './trino';
 export { MySQLDialect } from './mysql';
+export { DatabricksDialect } from './databricks';
 export { getDialect, registerDialect } from './dialect_map';
 export { getMalloyStandardFunctions } from './functions';
 export type { MalloyStandardFunctionImplementations } from './functions';

package/dist/dialect/index.js

@@ -22,7 +22,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TinyParser = exports.getMalloyStandardFunctions = exports.registerDialect = exports.getDialect = exports.MySQLDialect = exports.TrinoDialect = exports.SnowflakeDialect = exports.DuckDBDialect = exports.PostgresDialect = exports.StandardSQLDialect = exports.qtz = exports.Dialect = exports.sql = exports.literal = exports.variadicParam = exports.param = exports.spread = exports.maxScalar = exports.minAggregate = exports.minScalar = exports.overload = exports.makeParam = exports.anyExprType = exports.arg = void 0;
+exports.TinyParser = exports.getMalloyStandardFunctions = exports.registerDialect = exports.getDialect = exports.DatabricksDialect = exports.MySQLDialect = exports.TrinoDialect = exports.SnowflakeDialect = exports.DuckDBDialect = exports.PostgresDialect = exports.StandardSQLDialect = exports.qtz = exports.Dialect = exports.sql = exports.literal = exports.variadicParam = exports.param = exports.spread = exports.maxScalar = exports.minAggregate = exports.minScalar = exports.overload = exports.makeParam = exports.anyExprType = exports.arg = void 0;
 var util_1 = require("./functions/util");
 Object.defineProperty(exports, "arg", { enumerable: true, get: function () { return util_1.arg; } });
 Object.defineProperty(exports, "anyExprType", { enumerable: true, get: function () { return util_1.anyExprType; } });
@@ -51,6 +51,8 @@ var trino_1 = require("./trino");
 Object.defineProperty(exports, "TrinoDialect", { enumerable: true, get: function () { return trino_1.TrinoDialect; } });
 var mysql_1 = require("./mysql");
 Object.defineProperty(exports, "MySQLDialect", { enumerable: true, get: function () { return mysql_1.MySQLDialect; } });
+var databricks_1 = require("./databricks");
+Object.defineProperty(exports, "DatabricksDialect", { enumerable: true, get: function () { return databricks_1.DatabricksDialect; } });
 var dialect_map_1 = require("./dialect_map");
 Object.defineProperty(exports, "getDialect", { enumerable: true, get: function () { return dialect_map_1.getDialect; } });
 Object.defineProperty(exports, "registerDialect", { enumerable: true, get: function () { return dialect_map_1.registerDialect; } });

package/dist/dialect/mysql/mysql.d.ts

@@ -1,5 +1,5 @@
 import type { Sampling, MeasureTimeExpr, RegexMatchExpr, TimeExtractExpr, TypecastExpr, BasicAtomicTypeDef, AtomicTypeDef, TimestampTypeDef, ArrayLiteralNode, RecordLiteralNode } from '../../model/malloy_types';
-import type { BooleanTypeSupport, DialectFieldList, FieldReferenceType, OrderByClauseType, QueryInfo } from '../dialect';
+import type { BooleanTypeSupport, CompiledOrderBy, DialectFieldList, FieldReferenceType, OrderByClauseType, QueryInfo } from '../dialect';
 import { Dialect } from '../dialect';
 import type { DialectFunctionOverloadDef } from '../functions';
 export declare class MySQLDialect extends Dialect {
@@ -38,7 +38,7 @@ export declare class MySQLDialect extends Dialect {
     sqlGroupSetTable(groupSetCount: number): string;
     sqlAnyValue(_groupSet: number, fieldName: string): string;
     private mapFields;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/mysql/mysql.js

@@ -105,7 +105,7 @@ class MySQLDialect extends dialect_1.Dialect {
         this.unnestWithNumbers = false;
         this.defaultSampling = { rows: 50000 };
         this.supportUnnestArrayAgg = true;
-        this.supportsAggDistinct = true;
+        this.supportsAggDistinct = false;
         this.supportsCTEinCoorelatedSubQueries = true;
         this.supportsSafeCast = false;
         this.dontUnionIndex = false;
@@ -171,12 +171,13 @@ class MySQLDialect extends dialect_1.Dialect {
     }
     sqlAggregateTurtle(groupSet, fieldList, orderBy) {
         const separator = ',';
+        const orderByClause = orderBy ? this.sqlTurtleOrderByClause(orderBy) : '';
         let gc = `GROUP_CONCAT(
       IF(group_set=${groupSet},
         JSON_OBJECT(${this.mapFields(fieldList)})
         , null
         )
-      ${orderBy}
+      ${orderByClause}
       SEPARATOR '${separator}'
     )`;
         gc = `COALESCE(JSON_EXTRACT(CONCAT('[',${gc},']'),'$'),JSON_ARRAY())`;

package/dist/dialect/postgres/postgres.d.ts

@@ -1,6 +1,6 @@
 import type { Sampling, AtomicTypeDef, TypecastExpr, MeasureTimeExpr, BasicAtomicTypeDef, RecordLiteralNode, ArrayLiteralNode, TimeExtractExpr, TimestampUnit } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import { type DialectFieldList, type FieldReferenceType, type QueryInfo } from '../dialect';
+import { type CompiledOrderBy, type DialectFieldList, type FieldReferenceType, type QueryInfo } from '../dialect';
 import { PostgresBase } from '../pg_impl';
 export declare class PostgresDialect extends PostgresBase {
     name: string;
@@ -31,7 +31,7 @@ export declare class PostgresDialect extends PostgresBase {
     sqlGroupSetTable(groupSetCount: number): string;
     sqlAnyValue(groupSet: number, fieldName: string): string;
     mapFields(fieldList: DialectFieldList): string;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/postgres/postgres.js

@@ -57,6 +57,7 @@ const postgresToMalloyTypes = {
     'double precision': { type: 'number', numberType: 'float' },
     'timestamp without time zone': { type: 'timestamp' },
     'timestamp with time zone': { type: 'timestamptz' },
+    'timestamptz': { type: 'timestamptz' },
     'oid': { type: 'string' },
     'boolean': { type: 'boolean' },
     'timestamp': { type: 'timestamp' },
@@ -118,7 +119,8 @@ class PostgresDialect extends pg_impl_1.PostgresBase {
     }
     sqlAggregateTurtle(groupSet, fieldList, orderBy) {
         const fields = this.mapFields(fieldList);
-        return `COALESCE(TO_JSONB((ARRAY_AGG((SELECT TO_JSONB(__x) FROM (SELECT ${fields}\n  ) as __x) ${orderBy} ) FILTER (WHERE group_set=${groupSet}))),'[]'::JSONB)`;
+        const orderByClause = orderBy ? this.sqlTurtleOrderByClause(orderBy) : '';
+        return `COALESCE(TO_JSONB((ARRAY_AGG((SELECT TO_JSONB(__x) FROM (SELECT ${fields}\n  ) as __x) ${orderByClause} ) FILTER (WHERE group_set=${groupSet}))),'[]'::JSONB)`;
     }
     sqlAnyValueTurtle(groupSet, fieldList) {
         const fields = fieldList

package/dist/dialect/snowflake/snowflake.d.ts

@@ -1,6 +1,6 @@
 import type { Sampling, AtomicTypeDef, TimeExtractExpr, TypecastExpr, MeasureTimeExpr, RegexMatchExpr, BasicAtomicTypeDef, TimestampTypeDef, ArrayLiteralNode, RecordLiteralNode } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import type { DialectFieldList, FieldReferenceType, IntegerTypeMapping, QueryInfo } from '../dialect';
+import type { CompiledOrderBy, DialectFieldList, FieldReferenceType, IntegerTypeMapping, QueryInfo } from '../dialect';
 import { Dialect } from '../dialect';
 export declare class SnowflakeDialect extends Dialect {
     name: string;
@@ -32,7 +32,7 @@ export declare class SnowflakeDialect extends Dialect {
     sqlAnyValue(groupSet: number, fieldName: string): string;
     mapFields(fieldList: DialectFieldList): string;
     mapFieldsForObjectConstruct(fieldList: DialectFieldList): string;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/snowflake/snowflake.js

@@ -131,7 +131,9 @@ class SnowflakeDialect extends dialect_1.Dialect {
     }
     sqlAggregateTurtle(groupSet, fieldList, orderBy) {
         const fields = this.mapFieldsForObjectConstruct(fieldList);
-        const orderByClause = orderBy ? ` WITHIN GROUP (${orderBy})` : '';
+        const orderByClause = orderBy
+            ? ` WITHIN GROUP (${this.sqlTurtleOrderByClause(orderBy)})`
+            : '';
         const aggClause = `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN OBJECT_CONSTRUCT_KEEP_NULL(${fields}) END)${orderByClause}`;
         return `COALESCE(${aggClause}, [])`;
     }

package/dist/dialect/standardsql/standardsql.d.ts

@@ -1,7 +1,7 @@
 import type { Sampling, AtomicTypeDef, TimeExtractExpr, TypecastExpr, RegexMatchExpr, MeasureTimeExpr, BasicAtomicTypeDef, RecordLiteralNode, ArrayLiteralNode, TimestampUnit, TimestampTypeDef } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import type { DialectFieldList, IntegerTypeMapping, OrderByRequest, QueryInfo } from '../dialect';
-import { Dialect } from '../dialect';
+import type { CompiledOrderBy, DialectFieldList, IntegerTypeMapping, OrderByRequest, QueryInfo } from '../dialect';
+import { Dialect, type LateralJoinExpression } from '../dialect';
 export declare class StandardSQLDialect extends Dialect {
     name: string;
     experimental: boolean;
@@ -24,6 +24,7 @@ export declare class StandardSQLDialect extends Dialect {
     supportsNesting: boolean;
     cantPartitionWindowFunctionsOnExpressions: boolean;
     hasModOperator: boolean;
+    sqlLateralJoinBag(expressions: LateralJoinExpression[]): string;
     nestedArrays: boolean;
     supportsHyperLogLog: boolean;
     likeEscape: boolean;
@@ -36,7 +37,7 @@ export declare class StandardSQLDialect extends Dialect {
     sqlGroupSetTable(groupSetCount: number): string;
     sqlAnyValue(groupSet: number, fieldName: string): string;
     sqlOrderBy(orderTerms: string[], obr?: OrderByRequest): string;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/standardsql/standardsql.js

@@ -102,6 +102,10 @@ class StandardSQLDialect extends dialect_1.Dialect {
             { min: dialect_1.MIN_INT64, max: dialect_1.MAX_INT64, numberType: 'bigint' },
         ];
     }
+    sqlLateralJoinBag(expressions) {
+        const fields = expressions.map(e => `${e.sql} as ${e.name}`);
+        return `LEFT JOIN UNNEST([STRUCT(${fields.join(',\n')})]) as __lateral_join_bag\n`;
+    }
     quoteTablePath(tablePath) {
         return `\`${tablePath}\``;
     }
@@ -133,7 +137,8 @@ class StandardSQLDialect extends dialect_1.Dialect {
         const fields = fieldList
             .map(f => `\n  ${f.sqlExpression} as ${f.sqlOutputName}`)
             .join(', ');
-        return `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN STRUCT(${fields}\n  ) END IGNORE NULLS ${orderBy})`;
+        const orderByClause = orderBy ? this.sqlTurtleOrderByClause(orderBy) : '';
+        return `ARRAY_AGG(CASE WHEN group_set=${groupSet} THEN STRUCT(${fields}\n  ) END IGNORE NULLS ${orderByClause})`;
     }
     sqlAnyValueTurtle(groupSet, fieldList) {
         const fields = fieldList

package/dist/dialect/trino/trino.d.ts

@@ -1,6 +1,6 @@
 import type { Expr, Sampling, AtomicTypeDef, ATimestampTypeDef, TypecastExpr, RegexMatchExpr, MeasureTimeExpr, TimeExtractExpr, BasicAtomicTypeDef, RecordLiteralNode } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import type { DialectFieldList, OrderByClauseType, QueryInfo } from '../dialect';
+import type { CompiledOrderBy, DialectFieldList, OrderByClauseType, QueryInfo } from '../dialect';
 import { PostgresBase } from '../pg_impl';
 export declare class TrinoDialect extends PostgresBase {
     name: string;
@@ -35,7 +35,7 @@ export declare class TrinoDialect extends PostgresBase {
     exprToSQL(qi: QueryInfo, df: Expr): string | undefined;
     sqlAnyValue(groupSet: number, fieldName: string): string;
     buildTypeExpression(fieldList: DialectFieldList): string;
-    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: CompiledOrderBy[] | undefined): string;
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;

package/dist/dialect/trino/trino.js

@@ -232,7 +232,8 @@ class TrinoDialect extends pg_impl_1.PostgresBase {
     sqlAggregateTurtle(groupSet, fieldList, orderBy) {
         const expressions = fieldList.map(f => f.sqlExpression).join(',\n ');
         const definitions = this.buildTypeExpression(fieldList);
-        return `ARRAY_AGG(CAST(ROW(${expressions}) AS ROW(${definitions})) ${orderBy}) FILTER (WHERE group_set=${groupSet})`;
+        const orderByClause = orderBy ? this.sqlTurtleOrderByClause(orderBy) : '';
+        return `ARRAY_AGG(CAST(ROW(${expressions}) AS ROW(${definitions})) ${orderByClause}) FILTER (WHERE group_set=${groupSet})`;
     }
     sqlAnyValueTurtle(groupSet, fieldList) {
         const expressions = fieldList.map(f => f.sqlExpression).join(',\n ');

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { DuckDBDialect, StandardSQLDialect, TrinoDialect, PostgresDialect, SnowflakeDialect, MySQLDialect, registerDialect, arg, qtz, overload, minScalar, anyExprType, minAggregate, maxScalar, sql, makeParam, param, variadicParam, literal, spread, Dialect, TinyParser, } from './dialect';
+export { DuckDBDialect, StandardSQLDialect, TrinoDialect, PostgresDialect, SnowflakeDialect, MySQLDialect, DatabricksDialect, registerDialect, arg, qtz, overload, minScalar, anyExprType, minAggregate, maxScalar, sql, makeParam, param, variadicParam, literal, spread, Dialect, TinyParser, } from './dialect';
 export type { DialectFieldList, DialectFunctionOverloadDef, QueryInfo, MalloyStandardFunctionImplementations, DefinitionBlueprint, DefinitionBlueprintMap, OverloadedDefinitionBlueprint, TinyToken, } from './dialect';
 export type { QueryRecord, StructDef, TableSourceDef, SQLSourceDef, SourceDef, JoinFieldDef, NamedSourceDefs, MalloyQueryData, DateUnit, ExtractUnit, TimestampUnit, TemporalFieldType, QueryData, QueryValue, Expr, FilterCondition, Argument, Parameter, FieldDef, PipeSegment, QueryFieldDef, IndexFieldDef, TurtleDef, SearchValueMapResult, SearchIndexResult, ModelDef, Query, QueryResult, QueryResultDef, QueryRunStats, QueryScalar, NamedQueryDef, NamedModelObject, ExpressionType, FunctionDef, FunctionOverloadDef, FunctionParameterDef, ExpressionValueType, TypeDesc, FunctionParamTypeDesc, DocumentLocation, DocumentRange, DocumentPosition, Sampling, Annotation, BasicAtomicTypeDef, BasicAtomicDef, AtomicTypeDef, AtomicFieldDef, ArrayDef, ArrayTypeDef, RecordTypeDef, RepeatedRecordTypeDef, RecordDef, RepeatedRecordDef, RecordLiteralNode, StringLiteralNode, ArrayLiteralNode, SourceComponentInfo, DateLiteralNode, TimestampLiteralNode, TimestamptzLiteralNode, TimeLiteralExpr, TypecastExpr, BuildID, BuildManifest, BuildManifestEntry, } from './model';
 export { isSourceDef, isAtomic, isBasicAtomic, isCompoundArrayData, isJoined, isJoinedSource, isSamplingEnable, isSamplingPercent, isSamplingRows, isRepeatedRecord, isBasicArray, mkArrayDef, mkFieldDef, expressionIsAggregate, expressionIsAnalytic, expressionIsCalculation, expressionIsScalar, expressionIsUngroupedAggregate, indent, composeSQLExpr, isTimestampUnit, isDateUnit, constantExprToSQL, } from './model';

package/dist/index.js

@@ -33,8 +33,8 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Runtime = exports.Malloy = exports.Model = exports.MalloyTranslator = exports.malloyToQuery = exports.constantExprToSQL = exports.isDateUnit = exports.isTimestampUnit = exports.composeSQLExpr = exports.indent = exports.expressionIsUngroupedAggregate = exports.expressionIsScalar = exports.expressionIsCalculation = exports.expressionIsAnalytic = exports.expressionIsAggregate = exports.mkFieldDef = exports.mkArrayDef = exports.isBasicArray = exports.isRepeatedRecord = exports.isSamplingRows = exports.isSamplingPercent = exports.isSamplingEnable = exports.isJoinedSource = exports.isJoined = exports.isCompoundArrayData = exports.isBasicAtomic = exports.isAtomic = exports.isSourceDef = exports.TinyParser = exports.Dialect = exports.spread = exports.literal = exports.variadicParam = exports.param = exports.makeParam = exports.sql = exports.maxScalar = exports.minAggregate = exports.anyExprType = exports.minScalar = exports.overload = exports.qtz = exports.arg = exports.registerDialect = exports.MySQLDialect = exports.SnowflakeDialect = exports.PostgresDialect = exports.TrinoDialect = exports.StandardSQLDialect = exports.DuckDBDialect = void 0;
-exports.makeDigest = exports.EMPTY_BUILD_MANIFEST = exports.PersistSource = exports.annotationToTaglines = exports.annotationToTag = exports.sqlKey = exports.API = exports.sourceDefToSourceInfo = exports.modelDefToModelInfo = exports.toAsyncGenerator = exports.resolveValue = exports.isValueRef = exports.getRegisteredConnectionTypes = exports.getConnectionTypeDisplayName = exports.getConnectionProperties = exports.registerConnectionType = exports.MalloyConfig = exports.Manifest = exports.CacheManager = exports.InMemoryModelCache = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.AtomicFieldType = void 0;
+exports.Malloy = exports.Model = exports.MalloyTranslator = exports.malloyToQuery = exports.constantExprToSQL = exports.isDateUnit = exports.isTimestampUnit = exports.composeSQLExpr = exports.indent = exports.expressionIsUngroupedAggregate = exports.expressionIsScalar = exports.expressionIsCalculation = exports.expressionIsAnalytic = exports.expressionIsAggregate = exports.mkFieldDef = exports.mkArrayDef = exports.isBasicArray = exports.isRepeatedRecord = exports.isSamplingRows = exports.isSamplingPercent = exports.isSamplingEnable = exports.isJoinedSource = exports.isJoined = exports.isCompoundArrayData = exports.isBasicAtomic = exports.isAtomic = exports.isSourceDef = exports.TinyParser = exports.Dialect = exports.spread = exports.literal = exports.variadicParam = exports.param = exports.makeParam = exports.sql = exports.maxScalar = exports.minAggregate = exports.anyExprType = exports.minScalar = exports.overload = exports.qtz = exports.arg = exports.registerDialect = exports.DatabricksDialect = exports.MySQLDialect = exports.SnowflakeDialect = exports.PostgresDialect = exports.TrinoDialect = exports.StandardSQLDialect = exports.DuckDBDialect = void 0;
+exports.makeDigest = exports.EMPTY_BUILD_MANIFEST = exports.PersistSource = exports.annotationToTaglines = exports.annotationToTag = exports.sqlKey = exports.API = exports.sourceDefToSourceInfo = exports.modelDefToModelInfo = exports.toAsyncGenerator = exports.resolveValue = exports.isValueRef = exports.getRegisteredConnectionTypes = exports.getConnectionTypeDisplayName = exports.getConnectionProperties = exports.registerConnectionType = exports.MalloyConfig = exports.Manifest = exports.CacheManager = exports.InMemoryModelCache = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.AtomicFieldType = exports.Runtime = void 0;
 /*
  * Copyright 2023 Google LLC
  *
@@ -64,6 +64,7 @@ Object.defineProperty(exports, "TrinoDialect", { enumerable: true, get: function
 Object.defineProperty(exports, "PostgresDialect", { enumerable: true, get: function () { return dialect_1.PostgresDialect; } });
 Object.defineProperty(exports, "SnowflakeDialect", { enumerable: true, get: function () { return dialect_1.SnowflakeDialect; } });
 Object.defineProperty(exports, "MySQLDialect", { enumerable: true, get: function () { return dialect_1.MySQLDialect; } });
+Object.defineProperty(exports, "DatabricksDialect", { enumerable: true, get: function () { return dialect_1.DatabricksDialect; } });
 Object.defineProperty(exports, "registerDialect", { enumerable: true, get: function () { return dialect_1.registerDialect; } });
 Object.defineProperty(exports, "arg", { enumerable: true, get: function () { return dialect_1.arg; } });
 Object.defineProperty(exports, "qtz", { enumerable: true, get: function () { return dialect_1.qtz; } });

package/dist/lang/ast/query-elements/query-base.d.ts

@@ -5,5 +5,5 @@ export declare abstract class QueryBase extends MalloyElement {
     abstract queryComp(isRefOk: boolean): QueryComp;
     protected expandFieldUsage(inputSource: SourceDef, pipeline: PipeSegment[]): PipeSegment[];
     protected resolveCompositeSource(inputSource: SourceDef, pipeline: PipeSegment[]): SourceDef | undefined;
-    query(): Query;
+    query(isRefOk?: boolean): Query;
 }

package/dist/lang/ast/query-elements/query-base.js

@@ -57,8 +57,8 @@ class QueryBase extends malloy_element_1.MalloyElement {
         }
         return undefined;
     }
-    query() {
-        const { query } = this.queryComp(true);
+    query(isRefOk = true) {
+        const { query } = this.queryComp(isRefOk);
         return {
             ...query,
             pipeline: (0, query_utils_1.detectAndRemovePartialStages)(query.pipeline, this),

package/dist/lang/ast/query-elements/query-raw.d.ts

@@ -16,5 +16,5 @@ export declare class QueryRaw extends MalloyElement implements QueryElement {
     elementType: string;
     constructor(source: Source);
     queryComp(isRefOk: boolean): QueryComp;
-    query(): Query;
+    query(isRefOk?: boolean): Query;
 }

package/dist/lang/ast/query-elements/query-raw.js

@@ -57,8 +57,8 @@ class QueryRaw extends malloy_element_1.MalloyElement {
             inputStruct: structDef,
         };
     }
-    query() {
-        return this.queryComp(true).query;
+    query(isRefOk = true) {
+        return this.queryComp(isRefOk).query;
     }
 }
 exports.QueryRaw = QueryRaw;

package/dist/lang/ast/query-elements/query-reference.d.ts

@@ -13,5 +13,5 @@ export declare class QueryReference extends MalloyElement implements QueryElemen
     elementType: string;
     constructor(name: ModelEntryReference);
     queryComp(isRefOk: boolean): QueryComp;
-    query(): Query;
+    query(isRefOk?: boolean): Query;
 }

package/dist/lang/ast/query-elements/query-reference.js

@@ -71,8 +71,8 @@ class QueryReference extends malloy_element_1.MalloyElement {
         this.logError('non-query-used-as-query', `Illegal reference to '${this.name}', query expected`);
         return oops();
     }
-    query() {
-        return this.queryComp(true).query;
+    query(isRefOk = true) {
+        return this.queryComp(isRefOk).query;
     }
 }
 exports.QueryReference = QueryReference;

package/dist/lang/ast/sql-elements/sql-string.js

@@ -77,7 +77,7 @@ class SQLString extends malloy_element_1.MalloyElement {
                 // Not a source - try as a query
                 const queryObject = el.getQuery();
                 if (queryObject) {
-                    ret.push(queryObject.query());
+                    ret.push(queryObject.query(false));
                 }
                 else {
                     el.sqLog('failed-to-expand-sql-source', 'Cannot expand into a query');

package/dist/lang/ast/types/query-element.d.ts

@@ -3,6 +3,6 @@ import type { Query } from '../../../model/malloy_types';
 import type { QueryComp } from './query-comp';
 export interface QueryElement extends MalloyElement {
     queryComp(isRefOk: boolean): QueryComp;
-    query(): Query;
+    query(isRefOk?: boolean): Query;
 }
 export declare function isQueryElement(e: MalloyElement): e is QueryElement;

package/dist/lang/lib/Malloy/MalloyLexer.js

@@ -1,5 +1,5 @@
 "use strict";
-// Generated from MalloyLexer.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from src/lang/grammar/MalloyLexer.g4 by ANTLR 4.9.0-SNAPSHOT
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);

package/dist/lang/lib/Malloy/MalloyParser.js

@@ -1,5 +1,5 @@
 "use strict";
-// Generated from MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from src/lang/grammar/MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);

package/dist/lang/lib/Malloy/MalloyParserListener.js

@@ -1,4 +1,4 @@
 "use strict";
-// Generated from MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from src/lang/grammar/MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
 Object.defineProperty(exports, "__esModule", { value: true });
 //# sourceMappingURL=MalloyParserListener.js.map

package/dist/lang/lib/Malloy/MalloyParserVisitor.js

@@ -1,4 +1,4 @@
 "use strict";
-// Generated from MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from src/lang/grammar/MalloyParser.g4 by ANTLR 4.9.0-SNAPSHOT
 Object.defineProperty(exports, "__esModule", { value: true });
 //# sourceMappingURL=MalloyParserVisitor.js.map