@malloydata/malloy 0.0.412 → 0.0.414

package/dist/model/query_query.js

@@ -14,9 +14,31 @@ const field_instance_1 = require("./field_instance");
 const sql_compiled_1 = require("./sql_compiled");
 const source_def_utils_1 = require("./source_def_utils");
 const malloy_compile_error_1 = require("./malloy_compile_error");
+const nest_capability_1 = require("./nest-capability");
 function pathToCol(path) {
     return path.map(el => encodeURIComponent(el)).join('/');
 }
+/**
+ * Is this result's row limit applied by the ROW_NUMBER "shave" in
+ * generateSQLHavingLimit? A nested projection's is not: its limit slices its
+ * array-agg (see sqlAggregateTurtle), because a projection has no group-by
+ * columns for the shave to partition and order on.
+ */
+function usesShaveLimit(result) {
+    if (result.getLimit() === undefined)
+        return false;
+    const isNestedProjection = result.parent !== undefined && result.firstSegment.type === 'project';
+    return !isNestedProjection;
+}
+/** Is this an analytic (window) field -- a `calculate:`? */
+function fieldIsAnalytic(fi) {
+    return ((0, malloy_types_1.hasExpression)(fi.f.fieldDef) &&
+        (0, malloy_types_1.expressionIsAnalytic)(fi.f.fieldDef.expressionType));
+}
+// 1-based SELECT positions of the dimension columns, for `GROUP BY 1,2,...`.
+function groupByPositions(columns) {
+    return columns.flatMap((c, i) => (c.isDimension ? [i + 1] : []));
+}
 function pushDialectField(dl, f) {
     const { sqlExpression, sqlOutputName, rawName } = f;
     if ((0, malloy_types_1.isAtomic)(f.fieldDef)) {
@@ -886,15 +908,31 @@ class QueryQuery extends query_node_1.QueryField {
         }
         return s;
     }
+    /**
+     * The single-scan form, with no `group_set` fan-out: one `SELECT` over the
+     * scan, grouped by the dimensions. Used both for a query with no nests and --
+     * via canUseSingleGroupSetSQL -- for one whose only nests are single-stage
+     * grain-preserving projections, which become an array-agg in the same SELECT.
+     * That array-agg needs no `FILTER (WHERE group_set=N)` (every row is in the
+     * one group), and aggregate/window expressions must not gate themselves by
+     * group_set either, so emitsGroupSet is cleared (see FieldInstanceResultRoot).
+     */
     generateSimpleSQL(stageWriter) {
+        this.rootResult.emitsGroupSet = false;
         let s = '';
         s += 'SELECT \n';
         const fields = [];
-        for (const [name, field] of this.rootResult.allFields) {
-            const fi = field;
+        const outputPipelinedSQL = [];
+        for (const [name, fi] of this.rootResult.allFields) {
             const sqlName = this.parent.dialect.sqlQuoteIdentifier(name);
-            if (fi.fieldUsage.type === 'result') {
-                fields.push(` ${fi.generateExpression()} as ${sqlName}`);
+            if (fi instanceof field_instance_1.FieldInstanceField) {
+                if (fi.fieldUsage.type === 'result') {
+                    fields.push(` ${fi.generateExpression()} as ${sqlName}`);
+                }
+            }
+            else if (fi instanceof field_instance_1.FieldInstanceResult) {
+                const turtle = this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL, true);
+                fields.push(` ${turtle} as ${sqlName}`);
             }
         }
         s += (0, utils_1.indent)(fields.join(',\n')) + '\n';
@@ -921,29 +959,77 @@ class QueryQuery extends query_node_1.QueryField {
             s += `LIMIT ${this.firstSegment.limit}\n`;
         }
         this.resultStage = stageWriter.addStage(s);
+        // Consumes any pipelined turtle output. The nests that reach here are
+        // single-stage (canUseSingleGroupSetSQL), so outputPipelinedSQL is empty and
+        // this is a no-op; kept so the function stays correct if that guard widens.
+        this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter, []);
         return this.resultStage;
     }
+    /**
+     * Can this query take the single-group-set fast path? True when the query
+     * needs no group-set fan-out: it has nests (so it's "complex") but every one
+     * is a single-stage grain-preserving projection riding group_set 0, and there
+     * are no reduce nests, ungroupings, or totals (all of which would mint a
+     * second group set and push maxGroupSet above 0). Anything outside that shape
+     * falls back to the general demux path, which is correct for every case.
+     */
+    canUseSingleGroupSetSQL() {
+        if (this.maxGroupSet !== 0)
+            return false;
+        if (this.firstSegment.type !== 'reduce')
+            return false;
+        for (const [, fi] of this.rootResult.allFields) {
+            if (fi instanceof field_instance_1.FieldInstanceField) {
+                // A root-level `calculate:` (window) needs the general path's analytic
+                // handling: on dialects that can't partition a window on an expression
+                // (BigQuery) it rides a lateral-join bag this path doesn't build.
+                if (fieldIsAnalytic(fi))
+                    return false;
+            }
+            else if (fi instanceof field_instance_1.FieldInstanceResult) {
+                if (fi.firstSegment.type !== 'project')
+                    return false;
+                if (fi.turtleDef.pipeline.length !== 1)
+                    return false;
+                if (fi.getRepeatedResultType() !== 'nested')
+                    return false;
+                // A `calculate:` inside a `select:` would put a window function inside
+                // the array-agg, which no dialect allows; leave it on the general path.
+                for (const [, nestField] of fi.allFields) {
+                    if (nestField instanceof field_instance_1.FieldInstanceField &&
+                        fieldIsAnalytic(nestField)) {
+                        return false;
+                    }
+                }
+            }
+        }
+        return true;
+    }
     // This probably should be generated in a dialect independat way.
     //  but for now, it is just googleSQL.
-    generatePipelinedStages(outputPipelinedSQL, lastStageName, stageWriter) {
+    generatePipelinedStages(outputPipelinedSQL, lastStageName, stageWriter, 
+    // Every column `lastStageName` produces. The replaced turtle columns (in
+    // outputPipelinedSQL) are dropped and re-added by the pipeline expressions;
+    // everything else is carried through unchanged by its emitted name -- for a
+    // grouped stage that is the group-set-suffixed form (e.g. `f1__0`), not the
+    // final output name.
+    priorStageColumns) {
         if (outputPipelinedSQL.length === 0) {
             return lastStageName;
         }
+        const pipelinesSQL = outputPipelinedSQL
+            .map(o => `${o.pipelineSQL} as ${o.sqlFieldName}`)
+            .join(',\n');
         let retSQL;
         if (this.parent.dialect.supportsSelectReplace) {
-            const pipelinesSQL = outputPipelinedSQL
-                .map(o => `${o.pipelineSQL} as ${o.sqlFieldName}`)
-                .join(',\n');
             retSQL = `SELECT * replace (${pipelinesSQL}) FROM ${lastStageName}
         `;
         }
         else {
-            const pipelinesSQL = outputPipelinedSQL
-                .map(o => `${o.pipelineSQL} as ${o.sqlFieldName}`)
-                .join(',\n');
             const outputFields = outputPipelinedSQL.map(f => f.sqlFieldName);
-            const allFields = Array.from(this.rootResult.allFields.keys()).map(f => this.parent.dialect.sqlQuoteIdentifier(f));
-            const fields = allFields.filter(f => outputFields.indexOf(f) === -1);
+            const fields = priorStageColumns
+                .map(c => c.name)
+                .filter(name => outputFields.indexOf(name) === -1);
             retSQL = `SELECT ${fields.length > 0 ? fields.join(', ') + ',' : ''} ${pipelinesSQL} FROM ${lastStageName}`;
         }
         return stageWriter.addStage(retSQL);
@@ -978,12 +1064,19 @@ class QueryQuery extends query_node_1.QueryField {
                                 sql: exp,
                                 name: outputName,
                             });
-                            output.sql.push(outputFieldName);
+                            output.columns.push({
+                                sql: outputFieldName,
+                                name: outputName,
+                                isDimension: true,
+                            });
                             if (fi.f.fieldDef.type === 'number') {
                                 const outputNameString = this.parent.dialect.sqlQuoteIdentifier(`${name}__${resultSet.groupSet}_string`);
                                 const outputFieldNameString = `__lateral_join_bag.${outputNameString}`;
-                                output.sql.push(outputFieldNameString);
-                                output.dimensionIndexes.push(output.fieldIndex++);
+                                output.columns.push({
+                                    sql: outputFieldNameString,
+                                    name: outputNameString,
+                                    isDimension: true,
+                                });
                                 output.lateralJoinSQLExpressions.push({
                                     sql: `CAST(${exp} as STRING)`,
                                     name: outputNameString,
@@ -993,13 +1086,19 @@ class QueryQuery extends query_node_1.QueryField {
                         }
                         else {
                             // just treat it like a regular field.
-                            output.sql.push(`${exp} as ${outputName}`);
+                            output.columns.push({
+                                sql: `${exp} as ${outputName}`,
+                                name: outputName,
+                                isDimension: true,
+                            });
                         }
-                        output.dimensionIndexes.push(output.fieldIndex++);
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
-                        output.sql.push(`${exp} as ${outputName}`);
-                        output.fieldIndex++;
+                        output.columns.push({
+                            sql: `${exp} as ${outputName}`,
+                            name: outputName,
+                            isDimension: false,
+                        });
                     }
                 }
             }
@@ -1009,8 +1108,11 @@ class QueryQuery extends query_node_1.QueryField {
                 }
                 else if (fi.firstSegment.type === 'project') {
                     const s = this.generateTurtleSQL(fi, stageWriter, outputName, output.outputPipelinedSQL);
-                    output.sql.push(`${s} as ${outputName}`);
-                    output.fieldIndex++;
+                    output.columns.push({
+                        sql: `${s} as ${outputName}`,
+                        name: outputName,
+                        isDimension: false,
+                    });
                 }
             }
         }
@@ -1028,8 +1130,11 @@ class QueryQuery extends query_node_1.QueryField {
             }
             else {
                 resultSet.hasHaving = true;
-                output.sql.push(`CASE WHEN group_set=${resultSet.groupSet} THEN CASE WHEN ${having.sql()} THEN 0 ELSE 1 END END as __delete__${resultSet.groupSet}`);
-                output.fieldIndex++;
+                output.columns.push({
+                    sql: `CASE WHEN group_set=${resultSet.groupSet} THEN CASE WHEN ${having.sql()} THEN 0 ELSE 1 END END as __delete__${resultSet.groupSet}`,
+                    name: `__delete__${resultSet.groupSet}`,
+                    isDimension: false,
+                });
             }
         }
     }
@@ -1038,11 +1143,18 @@ class QueryQuery extends query_node_1.QueryField {
         for (const [, field] of resultStruct.allFields) {
             if (field.type === 'query') {
                 const fir = field;
-                const turtleWhere = this.generateSQLFilters(fir, 'where');
-                if (turtleWhere.present()) {
-                    const groupSets = fir.childGroups.join(',');
-                    wheres.add(`(group_set NOT IN (${groupSets})` +
-                        ` OR (group_set IN (${groupSets}) AND ${turtleWhere.sql()}))`);
+                // A projection first stage's `where:` folds into its array-agg
+                // condition (see generateTurtleSQL / sqlAggregateTurtle), not a
+                // scan-level WHERE: it rides its enclosing group_set, so a WHERE here
+                // can't isolate it from the parent, and its childGroups is empty (no
+                // group_set of its own) which would emit a broken `group_set IN ()`.
+                if (fir.firstSegment.type !== 'project') {
+                    const turtleWhere = this.generateSQLFilters(fir, 'where');
+                    if (turtleWhere.present()) {
+                        const groupSets = fir.childGroups.join(',');
+                        wheres.add(`(group_set NOT IN (${groupSets})` +
+                            ` OR (group_set IN (${groupSets}) AND ${turtleWhere.sql()}))`);
+                    }
                 }
                 wheres.addChain(this.generateSQLWhereChildren(fir));
             }
@@ -1079,11 +1191,11 @@ class QueryQuery extends query_node_1.QueryField {
         const partitionSQL = [];
         let hasAnyLimits = false;
         let hasResultsWithChildren = false;
-        const resultsWithHavingOrLimit = this.rootResult.selectStructs([], (result) => result.hasHaving || result.getLimit() !== undefined);
+        const resultsWithHavingOrLimit = this.rootResult.selectStructs([], (result) => result.hasHaving || usesShaveLimit(result));
         if (resultsWithHavingOrLimit.length > 0) {
             // loop through an generate the partitions
             for (const result of this.rootResult.selectStructs([], (_result) => true)) {
-                const hasLimit = result.getLimit() !== undefined;
+                const hasLimit = usesShaveLimit(result);
                 hasResultsWithChildren || (hasResultsWithChildren = result.childGroups.length > 1 && (hasLimit || result.hasHaving));
                 hasAnyLimits || (hasAnyLimits = hasLimit);
                 // find all the parent dimension names.
@@ -1201,9 +1313,7 @@ class QueryQuery extends query_node_1.QueryField {
         let from = this.generateSQLJoins(stageWriter);
         const wheres = this.generateSQLWhereTurtled();
         const f = {
-            dimensionIndexes: [1],
-            fieldIndex: 2,
-            sql: ['group_set'],
+            columns: [{ sql: 'group_set', name: 'group_set', isDimension: true }],
             lateralJoinSQLExpressions: [],
             groupsAggregated: [],
             outputPipelinedSQL: [],
@@ -1213,9 +1323,9 @@ class QueryQuery extends query_node_1.QueryField {
             throw new malloy_compile_error_1.MalloyCompileError("Cannot use 'select:' in a stage that contains nested views. " +
                 'Use `group_by:` or restructure the pipeline.', 'compiler-project-with-turtles', this.fieldDef.location);
         }
-        const groupBy = 'GROUP BY ' + f.dimensionIndexes.join(',') + '\n';
+        const groupBy = 'GROUP BY ' + groupByPositions(f.columns).join(',') + '\n';
         from += this.parent.dialect.sqlGroupSetTable(this.maxGroupSet) + '\n';
-        s += (0, utils_1.indent)(f.sql.join(',\n')) + '\n';
+        s += (0, utils_1.indent)(f.columns.map(c => c.sql).join(',\n')) + '\n';
         if (f.lateralJoinSQLExpressions.length > 0) {
             from += this.parent.dialect.sqlLateralJoinBag(f.lateralJoinSQLExpressions);
         }
@@ -1224,7 +1334,7 @@ class QueryQuery extends query_node_1.QueryField {
         const resultStage = stageWriter.addStage(s);
         // generate stages for havings and limits
         this.resultStage = this.generateSQLHavingLimit(stageWriter, resultStage);
-        this.resultStage = this.generatePipelinedStages(f.outputPipelinedSQL, this.resultStage, stageWriter);
+        this.resultStage = this.generatePipelinedStages(f.outputPipelinedSQL, this.resultStage, stageWriter, f.columns);
         return this.resultStage;
     }
     generateDepthNFields(depth, resultSet, output, stageWriter) {
@@ -1235,13 +1345,19 @@ class QueryQuery extends query_node_1.QueryField {
                 if (fi.fieldUsage.type === 'result') {
                     if ((0, query_node_1.isScalarField)(fi.f)) {
                         const exp = (0, utils_1.caseGroup)(resultSet.groupSet > 0 ? resultSet.childGroups : [], sqlFieldName);
-                        output.sql.push(`${exp} as ${sqlFieldName}`);
-                        output.dimensionIndexes.push(output.fieldIndex++);
+                        output.columns.push({
+                            sql: `${exp} as ${sqlFieldName}`,
+                            name: sqlFieldName,
+                            isDimension: true,
+                        });
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
                         const exp = this.parent.dialect.sqlAnyValue(resultSet.groupSet, sqlFieldName);
-                        output.sql.push(`${exp} as ${sqlFieldName}`);
-                        output.fieldIndex++;
+                        output.columns.push({
+                            sql: `${exp} as ${sqlFieldName}`,
+                            name: sqlFieldName,
+                            isDimension: false,
+                        });
                     }
                 }
             }
@@ -1256,8 +1372,11 @@ class QueryQuery extends query_node_1.QueryField {
                         toGroup: resultSet.groupSet,
                     });
                     groupsToMap.push(fi.groupSet);
-                    output.sql.push(`${s} as ${sqlFieldName}`);
-                    output.fieldIndex++;
+                    output.columns.push({
+                        sql: `${s} as ${sqlFieldName}`,
+                        name: sqlFieldName,
+                        isDimension: false,
+                    });
                 }
                 else {
                     this.generateDepthNFields(depth, fi, output, stageWriter);
@@ -1267,14 +1386,19 @@ class QueryQuery extends query_node_1.QueryField {
         if (output.groupsAggregated.length > 0) {
             if (!this.parent.dialect.hasLateralColumnAliasInSelect) {
                 // Safe to remap in-place; aggregate expressions won't see the alias.
-                // This runs on every recursive return but sql[0] replacement is
+                // This runs on every recursive return but column[0] replacement is
                 // idempotent — last call builds the full CASE from all accumulated
-                // groupsAggregated entries.
-                output.sql[0] = 'CASE ';
+                // groupsAggregated entries. The output is still named group_set.
+                let groupSetSQL = 'CASE ';
                 for (const m of output.groupsAggregated) {
-                    output.sql[0] += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
+                    groupSetSQL += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
                 }
-                output.sql[0] += 'ELSE group_set END as group_set';
+                groupSetSQL += 'ELSE group_set END as group_set';
+                output.columns[0] = {
+                    sql: groupSetSQL,
+                    name: 'group_set',
+                    isDimension: true,
+                };
             }
             // For hasLateralColumnAliasInSelect, the remap is handled in
             // generateSQLDepthN to avoid adding it multiple times from recursion.
@@ -1283,88 +1407,100 @@ class QueryQuery extends query_node_1.QueryField {
     generateSQLDepthN(depth, stageWriter, stageName) {
         let s = 'SELECT \n';
         const f = {
-            dimensionIndexes: [1],
-            fieldIndex: 2,
-            sql: ['group_set'],
+            columns: [{ sql: 'group_set', name: 'group_set', isDimension: true }],
             lateralJoinSQLExpressions: [],
             groupsAggregated: [],
             outputPipelinedSQL: [],
         };
         this.generateDepthNFields(depth, this.rootResult, f, stageWriter);
         // When column aliases are visible in the same SELECT (e.g. Databricks),
-        // replace sql[0] with the remap CASE aliased as __remapped_group_set
-        // (not group_set). This avoids shadowing the input group_set column
-        // that aggregate expressions reference. GROUP BY still uses position 1
-        // (the remapped value), so rows with different original group_sets
-        // collapse correctly. A follow-up CTE renames it to group_set.
+        // replace column[0]'s SQL with the remap CASE aliased as
+        // __remapped_group_set (not group_set). This avoids shadowing the input
+        // group_set column that aggregate expressions reference. GROUP BY still
+        // uses position 1 (the remapped value), so rows with different original
+        // group_sets collapse correctly. A follow-up CTE renames it back, so the
+        // column's carried-forward name stays group_set.
         const needsRemapStage = f.groupsAggregated.length > 0 &&
             this.parent.dialect.hasLateralColumnAliasInSelect;
         if (needsRemapStage) {
-            f.sql[0] = 'CASE ';
+            let groupSetSQL = 'CASE ';
             for (const m of f.groupsAggregated) {
-                f.sql[0] += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
+                groupSetSQL += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
             }
-            f.sql[0] += 'ELSE group_set END as __remapped_group_set';
+            groupSetSQL += 'ELSE group_set END as __remapped_group_set';
+            f.columns[0] = { sql: groupSetSQL, name: 'group_set', isDimension: true };
         }
-        s += (0, utils_1.indent)(f.sql.join(',\n')) + '\n';
+        s += (0, utils_1.indent)(f.columns.map(c => c.sql).join(',\n')) + '\n';
         s += `FROM ${stageName}\n`;
         const where = this.rootResult.eliminateComputeGroupsSQL();
         if (where.length > 0) {
             s += `WHERE ${where}\n`;
         }
-        if (f.dimensionIndexes.length > 0) {
-            s += `GROUP BY ${f.dimensionIndexes.join(',')}\n`;
+        const dimensionPositions = groupByPositions(f.columns);
+        if (dimensionPositions.length > 0) {
+            s += `GROUP BY ${dimensionPositions.join(',')}\n`;
         }
         this.resultStage = stageWriter.addStage(s);
         if (needsRemapStage) {
-            const cols = f.sql.slice(1).map(col => {
-                const asMatch = col.match(/as\s+(\S+)\s*$/i);
-                return asMatch ? asMatch[1] : '*';
-            });
+            const cols = f.columns.slice(1).map(c => c.name);
             const remapSQL = `SELECT \n${(0, utils_1.indent)(['__remapped_group_set as group_set', ...cols].join(',\n'))}\nFROM ${this.resultStage}\n`;
             this.resultStage = stageWriter.addStage(remapSQL);
         }
-        this.resultStage = this.generatePipelinedStages(f.outputPipelinedSQL, this.resultStage, stageWriter);
+        this.resultStage = this.generatePipelinedStages(f.outputPipelinedSQL, this.resultStage, stageWriter, f.columns);
         return this.resultStage;
     }
     genereateSQLCombineTurtles(stageWriter, stage0Name) {
         let s = 'SELECT\n';
-        const fieldsSQL = [];
-        let fieldIndex = 1;
+        // Columns this combine stage emits. Unlike the grouped stages above, these
+        // are the final output names (unsuffixed), not the group-set-suffixed form.
+        // A following pipelined stage carries them forward by name.
+        const columns = [];
         const outputPipelinedSQL = [];
-        const dimensionIndexes = [];
         for (const [name, fi] of this.rootResult.allFields) {
             const sqlName = this.parent.dialect.sqlQuoteIdentifier(name);
             if (fi instanceof field_instance_1.FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
                     if ((0, query_node_1.isScalarField)(fi.f)) {
-                        fieldsSQL.push(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`) + ` as ${sqlName}`);
-                        dimensionIndexes.push(fieldIndex++);
+                        columns.push({
+                            sql: this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`) + ` as ${sqlName}`,
+                            name: sqlName,
+                            isDimension: true,
+                        });
                     }
                     else if ((0, query_node_1.isBasicCalculation)(fi.f)) {
-                        fieldsSQL.push(this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName));
-                        fieldIndex++;
+                        columns.push({
+                            sql: this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName),
+                            name: sqlName,
+                            isDimension: false,
+                        });
                     }
                 }
             }
             else if (fi instanceof field_instance_1.FieldInstanceResult) {
                 if (fi.firstSegment.type === 'reduce') {
-                    fieldsSQL.push(`${this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL)} as ${sqlName}`);
-                    fieldIndex++;
+                    columns.push({
+                        sql: `${this.generateTurtleSQL(fi, stageWriter, sqlName, outputPipelinedSQL)} as ${sqlName}`,
+                        name: sqlName,
+                        isDimension: false,
+                    });
                 }
                 else if (fi.firstSegment.type === 'project') {
-                    fieldsSQL.push(this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName));
-                    fieldIndex++;
+                    columns.push({
+                        sql: this.parent.dialect.sqlAnyValueLastTurtle(this.parent.dialect.sqlQuoteIdentifier(`${name}__${this.rootResult.groupSet}`), this.rootResult.groupSet, sqlName),
+                        name: sqlName,
+                        isDimension: false,
+                    });
                 }
             }
         }
-        s += (0, utils_1.indent)(fieldsSQL.join(',\n')) + `\nFROM ${stage0Name}\n`;
+        s += (0, utils_1.indent)(columns.map(c => c.sql).join(',\n')) + `\nFROM ${stage0Name}\n`;
         const where = this.rootResult.eliminateComputeGroupsSQL();
         if (where.length > 0) {
             s += `WHERE ${where}\n`;
         }
-        if (dimensionIndexes.length > 0) {
-            s += `GROUP BY ${dimensionIndexes.join(',')}\n`;
+        const dimensionPositions = groupByPositions(columns);
+        if (dimensionPositions.length > 0) {
+            s += `GROUP BY ${dimensionPositions.join(',')}\n`;
         }
         // order by
         s += this.genereateSQLOrderBy(this.firstSegment, this.rootResult);
@@ -1373,7 +1509,7 @@ class QueryQuery extends query_node_1.QueryField {
             s += `LIMIT ${this.firstSegment.limit}\n`;
         }
         this.resultStage = stageWriter.addStage(s);
-        this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter);
+        this.resultStage = this.generatePipelinedStages(outputPipelinedSQL, this.resultStage, stageWriter, columns);
         return this.resultStage;
     }
     // create a simplified version of the StructDef for dialects.
@@ -1438,7 +1574,21 @@ class QueryQuery extends query_node_1.QueryField {
         }
         return dialectFieldList;
     }
-    generateTurtleSQL(resultStruct, stageWriter, sqlFieldName, outputPipelinedSQL) {
+    generateTurtleSQL(resultStruct, stageWriter, sqlFieldName, outputPipelinedSQL, 
+    // The single-group-set fast path (generateSingleGroupSetSQL) has no
+    // group_set column to filter on: every row is in the one group. The
+    // array-agg drops its `FILTER (WHERE group_set=N)` and keeps only a
+    // projection's own `where:`.
+    omitGroupSetFilter = false) {
+        // How to emit this nest's first stage — and the last line of defense before
+        // broken SQL for IR that didn't pass through the translator (cached,
+        // deserialized, or query-builder IR). Same `nestStrategy` the translator
+        // uses, so the two can't disagree.
+        const pipeline = resultStruct.turtleDef.pipeline;
+        const strategy = (0, nest_capability_1.nestStrategy)(pipeline[0], this.parent.dialect, pipeline.length > 1);
+        if (strategy.kind === 'unsupported') {
+            throw new malloy_compile_error_1.MalloyCompileError(`dialect '${this.parent.dialect.name}' cannot emit this nested query (${strategy.reason})`, strategy.reason, resultStruct.turtleDef.location);
+        }
         // calculate the ordering.
         const compiledOrderBy = [];
         let orderingField;
@@ -1483,7 +1633,24 @@ class QueryQuery extends query_node_1.QueryField {
             }
         }
         else {
-            ret = this.parent.dialect.sqlAggregateTurtle(resultStruct.groupSet, dialectFieldList, orderBy);
+            // A projection first stage applies its own `limit:` by slicing the
+            // array-agg (it has no group-by keys to ROW_NUMBER-shave on) and its
+            // `where:` by folding into the array-agg's group_set condition (it rides
+            // the enclosing group_set, so a scan-level WHERE can't isolate it). This
+            // is a first-stage question: if more stages follow, they consume this
+            // (limited/filtered) array via the recursion in generateTurtlePipelineSQL.
+            // A reduce first stage keeps the shave and the scan-level WHERE.
+            let limit;
+            let filterSQL;
+            if (strategy.kind === 'project') {
+                limit = strategy.limit;
+                const where = this.generateSQLFilters(resultStruct, 'where');
+                if (where.present()) {
+                    filterSQL = where.sql();
+                }
+            }
+            // (capability is guarded by nestStrategy at the top of this method)
+            ret = this.parent.dialect.sqlAggregateTurtle(omitGroupSetFilter ? undefined : resultStruct.groupSet, dialectFieldList, orderBy, limit, filterSQL);
         }
         // If the turtle is a pipeline, generate a UDF to compute it.
         const newStageWriter = new stage_writer_1.StageWriter(this.parent.dialect.supportsCTEinCoorelatedSubQueries, stageWriter);
@@ -1561,12 +1728,13 @@ class QueryQuery extends query_node_1.QueryField {
         this.maxGroupSet = r.nextGroupSetNumber - 1;
         this.rootResult.assignFieldsToGroups();
         (_a = this.rootResult).isComplexQuery || (_a.isComplexQuery = this.maxDepth > 0 || r.isComplex);
-        if (this.rootResult.isComplexQuery) {
+        // A complex query needs the group-set fan-out unless it has just one group
+        // set (only single-stage projection nests), in which case generateSimpleSQL
+        // emits the same no-fan-out form a non-nested query gets.
+        if (this.rootResult.isComplexQuery && !this.canUseSingleGroupSetSQL()) {
             return this.generateComplexSQL(stageWriter);
         }
-        else {
-            return this.generateSimpleSQL(stageWriter);
-        }
+        return this.generateSimpleSQL(stageWriter);
     }
     generateSQLFromPipeline(stageWriter) {
         this.parent.maybeEmitParameterizedSourceUsage();

package/dist/model/source_def_utils.d.ts

@@ -1,11 +1,11 @@
 /**
  * SourceDef Utilities for Persistence
  *
- * Key invariant: sourceID is ONLY assigned in DefineSource when a source
- * gets a name. Factory functions explicitly copy only the fields they need,
- * never using spread, to prevent accidental propagation of sourceID/extends.
- *
- * The `extends` property is set by callers when processing extend blocks.
+ * Key invariant: a source's identity fields (sourceID, referenceID, extends)
+ * are assigned in the translator (DefineSource / the reference path), never by
+ * these compiler factories. The factory functions explicitly copy only the
+ * fields they need, never using spread, so identity is not propagated onto a
+ * freshly built source.
  */
 import type { BuildID, FieldDef, GivenID, ModelDef, PersistableSourceDef, Query, QuerySourceDef, SourceDef, SourceID, SourceRegistryEntry, SourceRegistryValue, SQLPhraseSegment, SQLSourceDef, TableSourceDef } from './malloy_types';
 export declare function mkSourceID(name: string, url: string | undefined): SourceID;
@@ -30,13 +30,37 @@ export declare function mkSQLSourceDef(base: SourceDef, selectStr: string, selec
  */
 export declare function mkTableSourceDef(name: string, connection: string, tablePath: string, dialect: string, fields: FieldDef[]): TableSourceDef;
 /**
- * Resolve a sourceID to a SourceDef using the sourceRegistry.
+ * Resolve a SourceID (a source's own `sourceID`, or a `referenceID` pointing at
+ * one) to its SourceDef using the sourceRegistry, returning any kind of source.
  *
  * @param modelDef The model definition containing the registry
- * @param sourceID The sourceID to resolve
+ * @param sourceID The SourceID to resolve
  * @returns The SourceDef if found, undefined otherwise
  */
+export declare function resolveSourceRef(modelDef: ModelDef, sourceID: SourceID): SourceDef | undefined;
+/**
+ * Resolve a sourceID to a persistable SourceDef using the sourceRegistry.
+ *
+ * @param modelDef The model definition containing the registry
+ * @param sourceID The sourceID to resolve
+ * @returns The PersistableSourceDef if found, undefined otherwise
+ */
 export declare function resolveSourceID(modelDef: ModelDef, sourceID: SourceID): PersistableSourceDef | undefined;
+/** The namespace entry a source refers to (see `sourceNamespaceReference`). */
+export interface NamespaceReference {
+    /** The name this source is bound to in `modelDef.contents`. */
+    name: string;
+    /** The referenced source. */
+    source: SourceDef;
+}
+/**
+ * If `sd` was created as an unmodified reference to another source
+ * (`sd.referenceID` is set) and that source is present in this model's
+ * namespace, return it together with the name it goes by. Returns undefined
+ * when `sd` defines its own shape, or when the referenced source is not in this
+ * model's namespace (e.g. an imported source whose own target wasn't imported).
+ */
+export declare function sourceNamespaceReference(modelDef: ModelDef, sd: SourceDef): NamespaceReference | undefined;
 /**
  * Add an entry to the sourceRegistry.
  *