@malloydata/malloy 0.0.433 → 0.0.435

package/dist/model/query_query.js

@@ -34,14 +34,57 @@ function fieldIsAnalytic(fi) {
     return ((0, malloy_types_1.hasExpression)(fi.f.fieldDef) &&
         (0, malloy_types_1.expressionIsAnalytic)(fi.f.fieldDef.expressionType));
 }
-// 1-based SELECT positions of the dimension columns, for `GROUP BY 1,2,...`.
-function groupByPositions(columns) {
-    return columns.flatMap((c, i) => (c.isDimension ? [i + 1] : []));
+// `<expr> as <name>`
+function aliasedColumn(expr, name, role) {
+    var _a;
+    return {
+        sql: `${expr} as ${name}`,
+        expr,
+        name,
+        isDimension: role.isDimension,
+        constant: (_a = role.constant) !== null && _a !== void 0 ? _a : false,
+    };
+}
+// Does a field's expression read any column of its source?
+function readsColumn(fieldDef) {
+    if (!(0, malloy_types_1.hasExpression)(fieldDef)) {
+        return true;
+    }
+    for (const node of (0, utils_1.exprWalk)(fieldDef.e)) {
+        switch (node.node) {
+            case 'field':
+            case 'outputField':
+            case 'source-reference':
+            case 'genericSQLExpr':
+                return true;
+        }
+    }
+    return false;
+}
+// The dimension columns as GROUP BY terms: 1-based SELECT positions, or the
+// expressions for a dialect without ordinals in GROUP BY.
+function groupByTerms(dialect, columns) {
+    if (dialect.groupByClause === 'ordinal') {
+        return columns.flatMap((c, i) => (c.isDimension ? [`${i + 1}`] : []));
+    }
+    return columns.flatMap(c => (c.isDimension && !c.constant ? [c.expr] : []));
+}
+function sqlGroupBy(dialect, columns) {
+    const terms = groupByTerms(dialect, columns);
+    if (terms.length > 0) {
+        return `GROUP BY ${terms.join(',')}\n`;
+    }
+    // Dimensions which are all constants still make one group of the input
+    if (dialect.groupByClause === 'expression' &&
+        columns.some(c => c.isDimension)) {
+        return 'GROUP BY ()\n';
+    }
+    return '';
 }
 function pushDialectField(dl, f) {
-    const { sqlExpression, sqlOutputName, rawName } = f;
+    const { sqlExpression, rawName } = f;
     if ((0, malloy_types_1.isAtomic)(f.fieldDef)) {
-        dl.push({ typeDef: f.fieldDef, sqlExpression, sqlOutputName, rawName });
+        dl.push({ typeDef: f.fieldDef, sqlExpression, rawName });
     }
 }
 /** Query builder object. */
@@ -539,13 +582,23 @@ class QueryQuery extends query_node_1.QueryField {
             case 'virtual': {
                 const virtualMap = (_a = qs.prepareResultOptions) === null || _a === void 0 ? void 0 : _a.virtualMap;
                 const tablePath = (_b = virtualMap === null || virtualMap === void 0 ? void 0 : virtualMap.get(qs.structDef.connection)) === null || _b === void 0 ? void 0 : _b.get(qs.structDef.name);
-                if (!tablePath) {
+                if (tablePath === undefined) {
                     throw new malloy_compile_error_1.MalloyCompileError(`No virtual-map entry for virtual source '${qs.structDef.name}' ` +
                         `on connection '${qs.structDef.connection}'. ` +
-                        'Add a virtual-map entry via the `virtualMap` runtime option.', 'runtime-virtual-map-missing', qs.structDef.location);
+                        'Supply the entry in `virtualMap` when compiling or running the model.', 'runtime-virtual-map-missing', qs.structDef.location);
+                }
+                // Check the binding immediately before SQL substitution, using the
+                // executing query's dialect. Other entries in the map may be unused.
+                const dialect = this.parent.dialect;
+                const checked = dialect.sqlValidateTableName(tablePath);
+                if (!checked.ok || checked.canonical !== tablePath) {
+                    const reason = checked.ok
+                        ? `use the canonical path ${checked.canonical}`
+                        : checked.error;
+                    throw new malloy_compile_error_1.MalloyCompileError(`virtualMap entry '${qs.structDef.connection}.${qs.structDef.name}' ` +
+                        `has table path ${JSON.stringify(tablePath)}, which is not a ` +
+                        `canonical table path for ${dialect.name}: ${reason}`, 'runtime-virtual-map-invalid', qs.structDef.location);
                 }
-                // virtualMap entries are application-supplied — assumed already
-                // canonical SQL.
                 return tablePath;
             }
             case 'composite':
@@ -676,9 +729,7 @@ class QueryQuery extends query_node_1.QueryField {
                     return tempInstance.generateExpression();
                 });
             }
-            if (ji.children.length === 0 ||
-                conditions === undefined ||
-                !this.parent.dialect.supportsComplexFilteredSources) {
+            if (!ji.isSubquery()) {
                 // LTNOTE: need a check here to see the children's where: conditions are local
                 //  to the source and not to any of it's joined children.
                 //  In Presto, we're going to get a SQL error if in this case
@@ -700,9 +751,27 @@ class QueryQuery extends query_node_1.QueryField {
             else {
                 let select = `SELECT ${ji.alias}.*`;
                 let joins = '';
+                // Every join below this one lives inside the subquery, so every one of
+                // them has to be projected out of it, not just the direct children.
+                // A child which became a subquery of its own has already flattened its
+                // whole subtree into columns, so everything under it is read from that
+                // child rather than packed a second time.
+                const projectJoin = (j, from) => {
+                    const fieldList = this.joinProjectionFieldList(j);
+                    if (fieldList.length > 0) {
+                        select +=
+                            from === undefined
+                                ? `, ${this.parent.dialect.sqlSelectAliasAsStruct(j.alias, fieldList)} AS ${j.alias}`
+                                : `, ${from}.${j.alias} AS ${j.alias}`;
+                    }
+                    const readFrom = from !== null && from !== void 0 ? from : (j.isSubquery() ? j.alias : undefined);
+                    for (const below of j.children) {
+                        projectJoin(below, readFrom);
+                    }
+                };
                 for (const childJoin of ji.children) {
                     joins += this.generateSQLJoinBlock(stageWriter, childJoin, depth + 1);
-                    select += `, ${this.parent.dialect.sqlSelectAliasAsStruct(childJoin.alias, (0, utils_1.getDialectFieldList)(childJoin.queryStruct.structDef))} AS ${childJoin.alias}`;
+                    projectJoin(childJoin, undefined);
                 }
                 select += `\nFROM ${structSQL} AS ${ji.alias}\n${joins}\nWHERE ${conditions === null || conditions === void 0 ? void 0 : conditions.join(' AND ')}\n`;
                 s += `${matrixOperation} JOIN (\n${(0, utils_1.indent)(select)}) AS ${ji.alias}\n  ON ${onCondition}\n`;
@@ -746,6 +815,111 @@ class QueryQuery extends query_node_1.QueryField {
         }
         return s;
     }
+    /**
+     * Which column of which join a reference needs, or undefined when it needs
+     * none.  Walking `path` from `from`, the answer is the member of the
+     * innermost join along it -- a record is kept whole, since a reference into
+     * one is a reference to the column holding it.
+     *
+     * Two kinds of path need no column.  One ending at a computed field
+     * contributes that field's name, which is no column of the source; the
+     * columns the computation reads arrive as references of their own.  And a
+     * `having:` or `calculate:` names an *output* field, so its path is rooted
+     * in the query's output rather than its source and need not resolve here at
+     * all.  An output name which happens to match a source column resolves as
+     * though it were one, which costs nothing: a query's own source is never
+     * packed, only the joins beneath a filtered one.
+     */
+    packedColumnFor(from, path) {
+        let struct = from;
+        let alias = struct.getIdentifier();
+        let column = undefined;
+        for (const name of path) {
+            const child = struct.getChildByName(name);
+            if (child === undefined)
+                return undefined;
+            if (!(child instanceof query_node_1.QueryFieldStruct)) {
+                column !== null && column !== void 0 ? column : (column = name);
+                break;
+            }
+            struct = child.queryStruct;
+            if (struct.getJoinableParent() === struct) {
+                // A join: what the path named above it says nothing about what this
+                // one has to pack.
+                alias = struct.getIdentifier();
+                column = undefined;
+            }
+            else {
+                column !== null && column !== void 0 ? column : (column = name);
+            }
+        }
+        return column === undefined ? undefined : [alias, column];
+    }
+    /**
+     * Which columns of each join have to survive being packed into a struct:
+     * every column the segment's field usage names, plus each join's primary
+     * key, which the compiler reaches for as a distinct key without any
+     * expression in the query naming it.
+     */
+    packedColumnsByJoin() {
+        var _a;
+        if (this.packedColumns === undefined) {
+            const packed = new Map();
+            const needed = (found) => {
+                if (found === undefined)
+                    return;
+                const [alias, column] = found;
+                const columns = packed.get(alias);
+                if (columns) {
+                    columns.add(column);
+                }
+                else {
+                    packed.set(alias, new Set([column]));
+                }
+            };
+            const usage = (0, malloy_types_1.isRawSegment)(this.firstSegment)
+                ? []
+                : ((_a = this.firstSegment.expandedFieldUsage) !== null && _a !== void 0 ? _a : []);
+            for (const { path } of usage) {
+                needed(this.packedColumnFor(this.parent, path));
+            }
+            // What a primary key needs is what it reads: a computed key's name is
+            // not a column, and the columns its expression reads are named nowhere
+            // else in the query.
+            for (const ji of this.rootResult.joins.values()) {
+                const primaryKey = ji.queryStruct.primaryKey();
+                if (primaryKey === undefined)
+                    continue;
+                const keyDef = primaryKey.fieldDef;
+                if ((0, malloy_types_1.hasExpression)(keyDef)) {
+                    for (const { path } of (0, malloy_types_1.fieldUsageFrom)(keyDef.refSummary)) {
+                        needed(this.packedColumnFor(ji.queryStruct, path));
+                    }
+                }
+                else {
+                    needed(this.packedColumnFor(ji.queryStruct, [(0, malloy_types_1.activeName)(keyDef)]));
+                }
+            }
+            this.packedColumns = packed;
+        }
+        return this.packedColumns;
+    }
+    // What a join contributes to the SELECT of a filtered join's subquery: every
+    // column named outside the subquery, and the generated __distinct_key when
+    // the join needed one.
+    joinProjectionFieldList(ji) {
+        const packed = this.packedColumnsByJoin().get(ji.alias);
+        const dialect = ji.queryStruct.dialect;
+        const fieldList = (0, utils_1.getDialectFieldList)(ji.queryStruct.structDef, dialect).filter(d => packed === null || packed === void 0 ? void 0 : packed.has(d.rawName));
+        if (ji.makeUniqueKey) {
+            fieldList.push({
+                typeDef: { type: 'string' },
+                sqlExpression: dialect.sqlQuoteIdentifier('__distinct_key'),
+                rawName: '__distinct_key',
+            });
+        }
+        return fieldList;
+    }
     // BigQuery has wildcard psudo columns that are treated differently
     //  SELECT * FROM xxx doesn't include these psuedo columns but we need them so
     //  filters can get pushed down properly when generating a UNIQUE key.
@@ -773,9 +947,10 @@ class QueryQuery extends query_node_1.QueryField {
         // Joins
         let structSQL = this.getStructSourceSQL(qs, stageWriter);
         if ((0, malloy_types_1.isIndexSegment)(this.firstSegment)) {
-            structSQL = this.parent.dialect.sqlSampleTable(structSQL, this.firstSegment.sample);
+            structSQL = this.parent.dialect.sqlSampleTable(structSQL, this.firstSegment.sample, qs.structDef.type === 'table');
             if (this.firstSegment.sample) {
-                structSQL = stageWriter.addStage(`SELECT * from ${structSQL} as x limit 100000 `);
+                const d = this.parent.dialect;
+                structSQL = stageWriter.addStage(`SELECT * from ${structSQL} as x ${d.sqlOrderByLimit([], 100000)}`);
             }
         }
         if ((0, malloy_types_1.isBaseTable)(qs.structDef)) {
@@ -819,7 +994,6 @@ class QueryQuery extends query_node_1.QueryField {
         return result;
     }
     genereateSQLOrderBy(queryDef, resultStruct) {
-        let s = '';
         // Collect array joins for dialects that need explicit ordering.
         // Only for project queries - reduce queries aggregate rows so individual
         // row order doesn't matter, and we can't ORDER BY columns not in GROUP BY.
@@ -833,24 +1007,24 @@ class QueryQuery extends query_node_1.QueryField {
             // For project without explicit ordering, we still need array ordinality
             // ordering if the dialect requires it
             if (arrayJoins.length === 0) {
-                return ''; // No default ordering for project.
+                return []; // No default ordering for project.
             }
         }
         // Intermediate results (in a pipeline or join) that have no limit, don't need an orderby
         //  Some database don't have this optimization.
         if (this.fieldDef.pipeline.length > 1 && queryDef.limit === undefined) {
-            return '';
+            return [];
         }
         // ignore orderby if all aggregates.
         if (resultStruct.getRepeatedResultType() === 'inline_all_numbers') {
-            return '';
+            return [];
         }
         // if we are in the last stage of a query and the query is a subquery
         //  and has no limit, ORDER BY is superfluous
         if (this.isJoinedSubquery &&
             this.fieldDef.pipeline.length === 1 &&
             queryDef.limit === undefined) {
-            return '';
+            return [];
         }
         const orderBy = queryDef.orderBy || resultStruct.calculateDefaultOrderBy();
         const o = [];
@@ -893,10 +1067,7 @@ class QueryQuery extends query_node_1.QueryField {
         for (const aj of arrayJoins) {
             o.push(`${aj.alias}_outer.__row_id ASC`);
         }
-        if (o.length > 0) {
-            s = this.parent.dialect.sqlOrderBy(o, 'query') + '\n';
-        }
-        return s;
+        return o;
     }
     /**
      * The single-scan form, with no `group_set` fan-out: one `SELECT` over the
@@ -909,45 +1080,36 @@ class QueryQuery extends query_node_1.QueryField {
      */
     generateSimpleSQL(stageWriter) {
         this.rootResult.emitsGroupSet = false;
-        let s = '';
-        s += 'SELECT \n';
-        const fields = [];
+        const limit = !(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit
+            ? this.firstSegment.limit
+            : undefined;
+        let s = 'SELECT \n';
+        // A projection groups nothing; a reduce groups its scalar result fields.
+        const grouped = this.firstSegment.type === 'reduce';
+        const columns = [];
         const outputPipelinedSQL = [];
         for (const [name, fi] of this.rootResult.allFields) {
             const sqlName = this.parent.dialect.sqlQuoteIdentifier(name);
             if (fi instanceof field_instance_1.FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
-                    fields.push(` ${fi.generateExpression()} as ${sqlName}`);
+                    columns.push(aliasedColumn(fi.generateExpression(), sqlName, {
+                        isDimension: grouped && (0, query_node_1.isScalarField)(fi.f),
+                        constant: !readsColumn(fi.f.fieldDef),
+                    }));
                 }
             }
             else if (fi instanceof field_instance_1.FieldInstanceResult) {
                 const turtle = this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL, true);
-                fields.push(` ${turtle} as ${sqlName}`);
+                columns.push(aliasedColumn(turtle, sqlName, { isDimension: false }));
             }
         }
-        s += (0, utils_1.indent)(fields.join(',\n')) + '\n';
+        s += (0, utils_1.indent)(columns.map(c => ` ${c.sql}`).join(',\n')) + '\n';
         s += this.generateSQLJoins(stageWriter);
         s += this.generateSQLFilters(this.rootResult, 'where').sql('where');
-        // group by
-        if (this.firstSegment.type === 'reduce') {
-            const n = [];
-            for (const field of this.rootResult.fields()) {
-                const fi = field;
-                if (fi.fieldUsage.type === 'result' && (0, query_node_1.isScalarField)(fi.f)) {
-                    n.push(fi.fieldUsage.resultIndex.toString());
-                }
-            }
-            if (n.length > 0) {
-                s += `GROUP BY ${n.join(',')}\n`;
-            }
-        }
+        s += sqlGroupBy(this.parent.dialect, columns);
         s += this.generateSQLFilters(this.rootResult, 'having').sql('having');
-        // order by
-        s += this.genereateSQLOrderBy(this.firstSegment, this.rootResult);
-        // limit
-        if (!(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit) {
-            s += `LIMIT ${this.firstSegment.limit}\n`;
-        }
+        // order by, limit
+        s += this.parent.dialect.sqlOrderByLimit(this.genereateSQLOrderBy(this.firstSegment, this.rootResult), limit);
         this.resultStage = stageWriter.addStage(s);
         // Consumes any pipelined turtle output. The nests that reach here are
         // single-stage (canUseSingleGroupSetSQL), so outputPipelinedSQL is empty and
@@ -1059,6 +1221,7 @@ class QueryQuery extends query_node_1.QueryField {
                             });
                             output.columns.push({
                                 sql: outputFieldName,
+                                expr: outputFieldName,
                                 name: outputName,
                                 isDimension: true,
                             });
@@ -1067,6 +1230,7 @@ class QueryQuery extends query_node_1.QueryField {
                                 const outputFieldNameString = `__lateral_join_bag.${outputNameString}`;
                                 output.columns.push({
                                     sql: outputFieldNameString,
+                                    expr: outputFieldNameString,
                                     name: outputNameString,
                                     isDimension: true,
                                 });
@@ -1079,19 +1243,14 @@ class QueryQuery extends query_node_1.QueryField {
                         }
                         else {
                             // just treat it like a regular field.
-                            output.columns.push({
-                                sql: `${exp} as ${outputName}`,
-                                name: outputName,
+                            output.columns.push(aliasedColumn(exp, outputName, {
                                 isDimension: true,
-                            });
+                                constant: !readsColumn(fi.f.fieldDef),
+                            }));
                         }
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
-                        output.columns.push({
-                            sql: `${exp} as ${outputName}`,
-                            name: outputName,
-                            isDimension: false,
-                        });
+                        output.columns.push(aliasedColumn(exp, outputName, { isDimension: false }));
                     }
                 }
             }
@@ -1101,11 +1260,7 @@ class QueryQuery extends query_node_1.QueryField {
                 }
                 else if (fi.firstSegment.type === 'project') {
                     const s = this.generateTurtleSQL(fi, stageWriter, outputName, output.outputPipelinedSQL);
-                    output.columns.push({
-                        sql: `${s} as ${outputName}`,
-                        name: outputName,
-                        isDimension: false,
-                    });
+                    output.columns.push(aliasedColumn(s, outputName, { isDimension: false }));
                 }
             }
         }
@@ -1123,11 +1278,7 @@ class QueryQuery extends query_node_1.QueryField {
             }
             else {
                 resultSet.hasHaving = true;
-                output.columns.push({
-                    sql: `CASE WHEN group_set=${resultSet.groupSet} THEN CASE WHEN ${having.sql()} THEN 0 ELSE 1 END END as __delete__${resultSet.groupSet}`,
-                    name: `__delete__${resultSet.groupSet}`,
-                    isDimension: false,
-                });
+                output.columns.push(aliasedColumn(`CASE WHEN group_set=${resultSet.groupSet} THEN CASE WHEN ${having.sql()} THEN 0 ELSE 1 END END`, `__delete__${resultSet.groupSet}`, { isDimension: false }));
             }
         }
     }
@@ -1306,7 +1457,14 @@ class QueryQuery extends query_node_1.QueryField {
         let from = this.generateSQLJoins(stageWriter);
         const wheres = this.generateSQLWhereTurtled();
         const f = {
-            columns: [{ sql: 'group_set', name: 'group_set', isDimension: true }],
+            columns: [
+                {
+                    sql: 'group_set',
+                    expr: 'group_set',
+                    name: 'group_set',
+                    isDimension: true,
+                },
+            ],
             lateralJoinSQLExpressions: [],
             groupsAggregated: [],
             outputPipelinedSQL: [],
@@ -1316,7 +1474,7 @@ class QueryQuery extends query_node_1.QueryField {
             throw new malloy_compile_error_1.MalloyCompileError("Cannot use 'select:' in a stage that contains nested views. " +
                 'Use `group_by:` or restructure the pipeline.', 'compiler-project-with-turtles', this.fieldDef.location);
         }
-        const groupBy = 'GROUP BY ' + groupByPositions(f.columns).join(',') + '\n';
+        const groupBy = sqlGroupBy(this.parent.dialect, f.columns);
         from += this.parent.dialect.sqlGroupSetTable(this.maxGroupSet) + '\n';
         s += (0, utils_1.indent)(f.columns.map(c => c.sql).join(',\n')) + '\n';
         if (f.lateralJoinSQLExpressions.length > 0) {
@@ -1338,19 +1496,11 @@ class QueryQuery extends query_node_1.QueryField {
                 if (fi.fieldUsage.type === 'result') {
                     if ((0, query_node_1.isScalarField)(fi.f)) {
                         const exp = (0, utils_1.caseGroup)(resultSet.groupSet > 0 ? resultSet.childGroups : [], sqlFieldName);
-                        output.columns.push({
-                            sql: `${exp} as ${sqlFieldName}`,
-                            name: sqlFieldName,
-                            isDimension: true,
-                        });
+                        output.columns.push(aliasedColumn(exp, sqlFieldName, { isDimension: true }));
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
                         const exp = this.parent.dialect.sqlAnyValue(resultSet.groupSet, sqlFieldName);
-                        output.columns.push({
-                            sql: `${exp} as ${sqlFieldName}`,
-                            name: sqlFieldName,
-                            isDimension: false,
-                        });
+                        output.columns.push(aliasedColumn(exp, sqlFieldName, { isDimension: false }));
                     }
                 }
             }
@@ -1365,11 +1515,7 @@ class QueryQuery extends query_node_1.QueryField {
                         toGroup: resultSet.groupSet,
                     });
                     groupsToMap.push(fi.groupSet);
-                    output.columns.push({
-                        sql: `${s} as ${sqlFieldName}`,
-                        name: sqlFieldName,
-                        isDimension: false,
-                    });
+                    output.columns.push(aliasedColumn(s, sqlFieldName, { isDimension: false }));
                 }
                 else {
                     this.generateDepthNFields(depth, fi, output, stageWriter);
@@ -1386,12 +1532,10 @@ class QueryQuery extends query_node_1.QueryField {
                 for (const m of output.groupsAggregated) {
                     groupSetSQL += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
                 }
-                groupSetSQL += 'ELSE group_set END as group_set';
-                output.columns[0] = {
-                    sql: groupSetSQL,
-                    name: 'group_set',
+                groupSetSQL += 'ELSE group_set END';
+                output.columns[0] = aliasedColumn(groupSetSQL, 'group_set', {
                     isDimension: true,
-                };
+                });
             }
             // For hasLateralColumnAliasInSelect, the remap is handled in
             // generateSQLDepthN to avoid adding it multiple times from recursion.
@@ -1400,7 +1544,14 @@ class QueryQuery extends query_node_1.QueryField {
     generateSQLDepthN(depth, stageWriter, stageName) {
         let s = 'SELECT \n';
         const f = {
-            columns: [{ sql: 'group_set', name: 'group_set', isDimension: true }],
+            columns: [
+                {
+                    sql: 'group_set',
+                    expr: 'group_set',
+                    name: 'group_set',
+                    isDimension: true,
+                },
+            ],
             lateralJoinSQLExpressions: [],
             groupsAggregated: [],
             outputPipelinedSQL: [],
@@ -1420,8 +1571,13 @@ class QueryQuery extends query_node_1.QueryField {
             for (const m of f.groupsAggregated) {
                 groupSetSQL += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
             }
-            groupSetSQL += 'ELSE group_set END as __remapped_group_set';
-            f.columns[0] = { sql: groupSetSQL, name: 'group_set', isDimension: true };
+            groupSetSQL += 'ELSE group_set END';
+            f.columns[0] = {
+                sql: `${groupSetSQL} as __remapped_group_set`,
+                expr: groupSetSQL,
+                name: 'group_set',
+                isDimension: true,
+            };
         }
         s += (0, utils_1.indent)(f.columns.map(c => c.sql).join(',\n')) + '\n';
         s += `FROM ${stageName}\n`;
@@ -1429,10 +1585,7 @@ class QueryQuery extends query_node_1.QueryField {
         if (where.length > 0) {
             s += `WHERE ${where}\n`;
         }
-        const dimensionPositions = groupByPositions(f.columns);
-        if (dimensionPositions.length > 0) {
-            s += `GROUP BY ${dimensionPositions.join(',')}\n`;
-        }
+        s += sqlGroupBy(this.parent.dialect, f.columns);
         this.resultStage = stageWriter.addStage(s);
         if (needsRemapStage) {
             const cols = f.columns.slice(1).map(c => c.name);
@@ -1443,6 +1596,9 @@ class QueryQuery extends query_node_1.QueryField {
         return this.resultStage;
     }
     genereateSQLCombineTurtles(stageWriter, stage0Name) {
+        const limit = !(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit
+            ? this.firstSegment.limit
+            : undefined;
         let s = 'SELECT\n';
         // Columns this combine stage emits. Unlike the grouped stages above, these
         // are the final output names (unsuffixed), not the group-set-suffixed form.
@@ -1454,15 +1610,13 @@ class QueryQuery extends query_node_1.QueryField {
             if (fi instanceof field_instance_1.FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
                     if ((0, query_node_1.isScalarField)(fi.f)) {
-                        columns.push({
-                            sql: this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`) + ` as ${sqlName}`,
-                            name: sqlName,
-                            isDimension: true,
-                        });
+                        columns.push(aliasedColumn(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), sqlName, { isDimension: true }));
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
+                        const lastTurtle = this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName);
                         columns.push({
-                            sql: this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName),
+                            sql: lastTurtle,
+                            expr: lastTurtle,
                             name: sqlName,
                             isDimension: false,
                         });
@@ -1471,15 +1625,13 @@ class QueryQuery extends query_node_1.QueryField {
             }
             else if (fi instanceof field_instance_1.FieldInstanceResult) {
                 if (fi.firstSegment.type === 'reduce') {
-                    columns.push({
-                        sql: `${this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL)} as ${sqlName}`,
-                        name: sqlName,
-                        isDimension: false,
-                    });
+                    columns.push(aliasedColumn(this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL), sqlName, { isDimension: false }));
                 }
                 else if (fi.firstSegment.type === 'project') {
+                    const lastTurtle = this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName);
                     columns.push({
-                        sql: this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName),
+                        sql: lastTurtle,
+                        expr: lastTurtle,
                         name: sqlName,
                         isDimension: false,
                     });
@@ -1491,27 +1643,18 @@ class QueryQuery extends query_node_1.QueryField {
         if (where.length > 0) {
             s += `WHERE ${where}\n`;
         }
-        const dimensionPositions = groupByPositions(columns);
-        if (dimensionPositions.length > 0) {
-            s += `GROUP BY ${dimensionPositions.join(',')}\n`;
-        }
-        // order by
-        const orderBySQL = this.genereateSQLOrderBy(this.firstSegment, this.rootResult);
-        s += orderBySQL;
-        // limit
-        if (!(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit) {
-            s += `LIMIT ${this.firstSegment.limit}\n`;
-        }
+        s += sqlGroupBy(this.parent.dialect, columns);
+        // order by, limit
+        const orderTerms = this.genereateSQLOrderBy(this.firstSegment, this.rootResult);
+        s += this.parent.dialect.sqlOrderByLimit(orderTerms, limit);
         this.resultStage = stageWriter.addStage(s);
-        this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter, columns, orderBySQL);
+        this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter, columns, this.parent.dialect.sqlOrderByLimit(orderTerms, undefined));
         return this.resultStage;
     }
     // create a simplified version of the StructDef for dialects.
     buildDialectFieldList(resultStruct) {
         const dialectFieldList = [];
         for (const [name, field] of resultStruct.allFields) {
-            const sqlName = this.parent.dialect.sqlQuoteIdentifier(name);
-            //
             if (resultStruct.firstSegment.type === 'reduce' &&
                 field instanceof field_instance_1.FieldInstanceResult) {
                 const { structDef, repeatedResultType } = this.generateTurtlePipelineSQL(field, new stage_writer_1.StageWriter(true, undefined), '<nosource>');
@@ -1527,7 +1670,6 @@ class QueryQuery extends query_node_1.QueryField {
                         typeDef: multiLineNest,
                         sqlExpression: this.parent.dialect.sqlQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
                         rawName: name,
-                        sqlOutputName: sqlName,
                     });
                 }
                 else {
@@ -1541,7 +1683,6 @@ class QueryQuery extends query_node_1.QueryField {
                         typeDef: oneLineNest,
                         sqlExpression: this.parent.dialect.sqlQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
                         rawName: name,
-                        sqlOutputName: sqlName,
                     });
                 }
             }
@@ -1552,7 +1693,6 @@ class QueryQuery extends query_node_1.QueryField {
                     fieldDef: field.f.fieldDef,
                     sqlExpression: this.parent.dialect.sqlQuoteIdentifier(`${name}__${resultStruct.groupSet}`),
                     rawName: name,
-                    sqlOutputName: sqlName,
                 });
             }
             else if (resultStruct.firstSegment.type === 'project' &&
@@ -1562,7 +1702,6 @@ class QueryQuery extends query_node_1.QueryField {
                     fieldDef: field.f.fieldDef,
                     sqlExpression: field.generateExpression(),
                     rawName: name,
-                    sqlOutputName: sqlName,
                 });
             }
         }
@@ -1684,7 +1823,7 @@ class QueryQuery extends query_node_1.QueryField {
             const inputStruct = {
                 type: 'nest_source',
                 name: '~pipe~',
-                pipeSQL: this.parent.dialect.sqlUnnestPipelineHead(repeatedResultType === 'inline_all_numbers', sourceSQLExpression, (0, utils_1.getDialectFieldList)(structDef)),
+                pipeSQL: this.parent.dialect.sqlUnnestPipelineHead(repeatedResultType === 'inline_all_numbers', sourceSQLExpression, (0, utils_1.getDialectFieldList)(structDef, this.parent.dialect)),
                 fields: structDef.fields,
                 connection: structDef.connection,
                 dialect: structDef.dialect,
@@ -1822,30 +1961,37 @@ class QueryQueryIndexStage extends QueryQuery {
                 fields.push({ name, path, type: fi.f.fieldDef.type, expression });
             }
         }
-        let s = 'SELECT\n  group_set,\n';
-        s += '  CASE group_set\n';
-        for (let i = 0; i < fields.length; i++) {
-            s += `    WHEN ${i} THEN '${fields[i].name}'\n`;
-        }
-        s += `  END as ${fieldNameColumn},\n`;
-        s += '  CASE group_set\n';
-        for (let i = 0; i < fields.length; i++) {
-            const path = pathToCol(fields[i].path);
-            s += `    WHEN ${i} THEN '${path}'\n`;
-        }
-        s += `  END as ${fieldPathColumn},\n`;
-        s += '  CASE group_set\n';
-        for (let i = 0; i < fields.length; i++) {
-            s += `    WHEN ${i} THEN '${fields[i].type}'\n`;
-        }
-        s += `  END as ${fieldTypeColumn},`;
-        s += `  CASE group_set WHEN 99999 THEN ${dialect.castToString('NULL')}\n`;
-        for (let i = 0; i < fields.length; i++) {
-            if (fields[i].type === 'string') {
-                s += `    WHEN ${i} THEN ${fields[i].expression}\n`;
+        // The grouped columns: group_set and one CASE per index column.
+        const caseOnGroupSet = (when, firstWhen) => {
+            let c = 'CASE group_set\n';
+            if (firstWhen !== undefined) {
+                c += `    ${firstWhen}\n`;
             }
-        }
-        s += `  END as ${fieldValueColumn},\n`;
+            for (let i = 0; i < fields.length; i++) {
+                const value = when(i);
+                if (value !== undefined) {
+                    c += `    WHEN ${i} THEN ${value}\n`;
+                }
+            }
+            return c + '  END';
+        };
+        const grouped = [
+            {
+                sql: 'group_set',
+                expr: 'group_set',
+                name: 'group_set',
+                isDimension: true,
+            },
+            aliasedColumn(caseOnGroupSet(i => `'${fields[i].name}'`), fieldNameColumn, { isDimension: true }),
+            aliasedColumn(caseOnGroupSet(i => `'${pathToCol(fields[i].path)}'`), fieldPathColumn, { isDimension: true }),
+            aliasedColumn(caseOnGroupSet(i => `'${fields[i].type}'`), fieldTypeColumn, { isDimension: true }),
+            aliasedColumn(caseOnGroupSet(i => (fields[i].type === 'string' ? fields[i].expression : undefined), `WHEN 99999 THEN ${dialect.castToString('NULL')}`), fieldValueColumn, { isDimension: true }),
+        ];
+        const limit = !(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit
+            ? this.firstSegment.limit
+            : undefined;
+        let s = 'SELECT\n';
+        s += grouped.map(c => `  ${c.sql},\n`).join('');
         s += ` ${measureSQL} as ${weightColumn},\n`;
         // just in case we don't have any field types, force the case statement to have at least one value.
         s += "  CASE group_set\n    WHEN 99999 THEN ''";
@@ -1868,11 +2014,8 @@ class QueryQueryIndexStage extends QueryQuery {
         s += this.generateSQLJoins(stageWriter);
         s += dialect.sqlGroupSetTable(fields.length) + '\n';
         s += this.generateSQLFilters(this.rootResult, 'where').sql('where');
-        s += 'GROUP BY 1,2,3,4,5\n';
-        // limit
-        if (!(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit) {
-            s += `LIMIT ${this.firstSegment.limit}\n`;
-        }
+        s += sqlGroupBy(dialect, grouped);
+        s += dialect.sqlOrderByLimit([], limit);
         // console.log(s);
         const resultStage = stageWriter.addStage(s);
         this.resultStage = stageWriter.addStage(`SELECT