@malloydata/malloy 0.0.219 → 0.0.220-dev241204170603

package/dist/model/malloy_query.js

@@ -173,13 +173,6 @@ class StageWriter {
         }
     }
 }
-// type QueryNodeType =
-//   | "abstract"
-//   | "dimension"
-//   | "measure"
-//   | "query"
-//   | "turtle"
-//   | "struct";
 class GenerateState {
     constructor() {
         this.totalGroupSet = -1;
@@ -234,6 +227,9 @@ class QueryField extends QueryNode {
         }
         return parent;
     }
+    isAtomic() {
+        return (0, malloy_types_1.isAtomic)(this.fieldDef);
+    }
     caseGroup(groupSets, s) {
         if (groupSets.length === 0) {
             return s;
@@ -900,33 +896,47 @@ class QueryField extends QueryNode {
         return (parentDef.type === 'array' &&
             parentDef.elementTypeDef.type !== 'record_element');
     }
-    getExpr() {
+    generateExpression(resultSet) {
+        // If the field itself is an expression, generate it ..
         if ((0, malloy_types_1.hasExpression)(this.fieldDef)) {
-            return this.fieldDef.e;
+            return this.exprToSQL(resultSet, this.parent, this.fieldDef.e);
         }
-        const pType = this.parent.structDef.type;
-        return {
-            node: 'genericSQLExpr',
-            kids: { args: [] },
-            src: [
-                this.parent.dialect.sqlFieldReference(this.parent.getSQLIdentifier(), this.fieldDef.name, this.fieldDef.type, pType === 'record' || pType === 'array' || pType === 'nest_source', (0, malloy_types_1.isScalarArray)(this.parent.structDef)),
-            ],
-        };
-    }
-    generateExpression(resultSet) {
-        return this.exprToSQL(resultSet, this.parent, this.getExpr());
+        // The field itself is not an expression, so we would like
+        // to generate a dotted path to the field, EXCEPT ...
+        // some of the steps in the dotting might not exist
+        // in the namespace of their parent, but rather be record
+        // expressions which should be evaluated in the namespace
+        // of their parent.
+        // So we walk the tree and ask each one to compute itself
+        for (let ancestor = this.parent; ancestor !== undefined; ancestor = ancestor.parent) {
+            if (ancestor.structDef.type === 'record' &&
+                (0, malloy_types_1.hasExpression)(ancestor.structDef) &&
+                ancestor.recordAlias === undefined) {
+                if (!ancestor.parent) {
+                    throw new Error('Inconcievable record ancestor with expression but no parent');
+                }
+                const aliasValue = this.exprToSQL(resultSet, ancestor.parent, ancestor.structDef.e);
+                ancestor.informOfAliasValue(aliasValue);
+            }
+        }
+        return this.parent.sqlChildReference(this.fieldDef.name, this.parent.structDef.type === 'record'
+            ? {
+                result: resultSet,
+                field: this,
+            }
+            : undefined);
     }
 }
 function isCalculatedField(f) {
-    return f instanceof QueryAtomicField && f.isCalculated();
+    return f instanceof AbstractQueryAtomic && f.isCalculated();
 }
 function isAggregateField(f) {
-    return f instanceof QueryAtomicField && f.isAggregate();
+    return f instanceof AbstractQueryAtomic && f.isAggregate();
 }
 function isScalarField(f) {
-    return f instanceof QueryAtomicField && !f.isCalculated() && !f.isAggregate();
+    return (f instanceof AbstractQueryAtomic && !f.isCalculated() && !f.isAggregate());
 }
-class QueryAtomicField extends QueryField {
+class AbstractQueryAtomic extends QueryField {
     constructor(fieldDef, parent, refId) {
         super(fieldDef, parent, refId);
         this.fieldDef = fieldDef; // wish I didn't have to do this
@@ -948,28 +958,32 @@ class QueryAtomicField extends QueryField {
     hasExpression() {
         return (0, malloy_types_1.hasExpression)(this.fieldDef);
     }
+    isAtomic() {
+        return true;
+    }
 }
 // class QueryMeasure extends QueryField {}
-class QueryFieldString extends QueryAtomicField {
+class QueryFieldString extends AbstractQueryAtomic {
 }
-class QueryFieldNumber extends QueryAtomicField {
+class QueryFieldNumber extends AbstractQueryAtomic {
 }
-class QueryFieldBoolean extends QueryAtomicField {
+class QueryFieldBoolean extends AbstractQueryAtomic {
 }
-class QueryFieldJSON extends QueryAtomicField {
+class QueryFieldJSON extends AbstractQueryAtomic {
 }
-class QueryFieldUnsupported extends QueryAtomicField {
+class QueryFieldUnsupported extends AbstractQueryAtomic {
 }
-class QueryFieldDate extends QueryAtomicField {
+class QueryFieldDate extends AbstractQueryAtomic {
     generateExpression(resultSet) {
         const fd = this.fieldDef;
+        const superExpr = super.generateExpression(resultSet);
         if (!fd.timeframe) {
-            return super.generateExpression(resultSet);
+            return superExpr;
         }
         else {
             const truncated = {
                 node: 'trunc',
-                e: (0, malloy_types_1.mkTemporal)(this.getExpr(), 'date'),
+                e: (0, malloy_types_1.mkTemporal)({ node: 'genericSQLExpr', src: [superExpr], kids: { args: [] } }, 'date'),
                 units: fd.timeframe,
             };
             return this.exprToSQL(resultSet, this.parent, truncated);
@@ -985,7 +999,7 @@ class QueryFieldDate extends QueryAtomicField {
         return new QueryFieldDate(fieldDef, this.parent);
     }
 }
-class QueryFieldTimestamp extends QueryAtomicField {
+class QueryFieldTimestamp extends AbstractQueryAtomic {
     // clone ourselves on demand as a timeframe.
     getChildByName(name) {
         const fieldDef = {
@@ -996,7 +1010,7 @@ class QueryFieldTimestamp extends QueryAtomicField {
         return new QueryFieldTimestamp(fieldDef, this.parent);
     }
 }
-class QueryFieldDistinctKey extends QueryAtomicField {
+class QueryFieldDistinctKey extends AbstractQueryAtomic {
     generateExpression(resultSet) {
         var _a;
         if (this.parent.primaryKey()) {
@@ -1006,11 +1020,11 @@ class QueryFieldDistinctKey extends QueryAtomicField {
         else if (this.parent.structDef.type === 'array') {
             const parentKey = (_a = this.parent.parent) === null || _a === void 0 ? void 0 : _a.getDistinctKey().generateExpression(resultSet);
             return this.parent.dialect.sqlMakeUnnestKey(parentKey || '', // shouldn't have to do this...
-            this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), '__row_id', 'string', true, false));
+            this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), 'table', '__row_id', 'string'));
         }
         else {
             // return this.parent.getIdentifier() + "." + "__distinct_key";
-            return this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), '__distinct_key', 'string', false, false);
+            return this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), 'table', '__distinct_key', 'string');
         }
     }
     includeInWildcard() {
@@ -1219,7 +1233,7 @@ class FieldInstanceResult {
                 if (isScalarField(f.f)) {
                     return 'nested';
                 }
-                if (f.f instanceof QueryStruct) {
+                if (f.f instanceof QueryFieldStruct) {
                     ret = 'inline';
                 }
             }
@@ -1313,7 +1327,9 @@ class FieldInstanceResult {
     }
     findJoins(query) {
         for (const dim of this.fields()) {
-            this.addStructToJoin(dim.f.getJoinableParent(), query, dim.f.uniqueKeyPossibleUse(), []);
+            if (!(dim.f instanceof QueryFieldStruct)) {
+                this.addStructToJoin(dim.f.getJoinableParent(), query, dim.f.uniqueKeyPossibleUse(), []);
+            }
         }
         for (const s of this.structs()) {
             s.findJoins(query);
@@ -1526,9 +1542,6 @@ class JoinInstance {
         return getDialectFieldList(this.queryStruct.structDef);
     }
 }
-/** nested query */
-class QueryTurtle extends QueryField {
-}
 /**
  * Used by the translator to get the output StructDef of a pipe segment
  *
@@ -1622,20 +1635,30 @@ class QueryQuery extends QueryField {
         return { as, field };
     }
     addDependantPath(resultStruct, context, path, uniqueKeyPossibleUse, joinStack) {
-        const node = context.getFieldByName(path);
-        let struct;
-        if (node instanceof QueryField) {
-            struct = node.parent;
-        }
-        else if (node instanceof QueryStruct) {
-            struct = node;
-        }
-        else {
-            throw new Error('Internal Error:  Unknown object type');
+        if (path.length === 0) {
+            return;
         }
+        const node = context.getFieldByName(path);
+        const joinableParent = node instanceof QueryFieldStruct
+            ? node.queryStruct.getJoinableParent()
+            : node.parent.getJoinableParent();
         resultStruct
             .root()
-            .addStructToJoin(struct.getJoinableParent(), this, uniqueKeyPossibleUse, joinStack);
+            .addStructToJoin(joinableParent, this, uniqueKeyPossibleUse, joinStack);
+    }
+    findRecordAliases(context, path) {
+        for (const seg of path) {
+            const field = context.getFieldByName([seg]);
+            if (field instanceof QueryFieldStruct) {
+                const qs = field.queryStruct;
+                if (qs.structDef.type === 'record' &&
+                    (0, malloy_types_1.hasExpression)(qs.structDef) &&
+                    qs.parent) {
+                    qs.informOfAliasValue(this.exprToSQL(this.rootResult, qs.parent, qs.structDef.e));
+                }
+                context = qs;
+            }
+        }
     }
     addDependantExpr(resultStruct, context, e, joinStack) {
         var _a;
@@ -1679,6 +1702,7 @@ class QueryQuery extends QueryField {
                 }
             }
             if (expr.node === 'field') {
+                this.findRecordAliases(context, expr.path);
                 const field = context.getDimensionOrMeasureByName(expr.path);
                 if ((0, malloy_types_1.hasExpression)(field.fieldDef)) {
                     this.addDependantExpr(resultStruct, field.parent, field.fieldDef.e, joinStack);
@@ -1692,6 +1716,7 @@ class QueryQuery extends QueryField {
             else if (expr.node === 'aggregate') {
                 if ((0, malloy_types_1.isAsymmetricExpr)(expr)) {
                     if (expr.structPath) {
+                        this.findRecordAliases(context, expr.structPath);
                         this.addDependantPath(resultStruct, context, expr.structPath, expr.function, joinStack);
                     }
                     else {
@@ -1719,7 +1744,7 @@ class QueryQuery extends QueryField {
         let resultIndex = 1;
         for (const f of this.getSegmentFields(resultStruct)) {
             const { as, field } = this.expandField(f);
-            if (field instanceof QueryTurtle || field instanceof QueryQuery) {
+            if (field instanceof QueryQuery) {
                 if (this.firstSegment.type === 'project') {
                     throw new Error(`Nested views cannot be used in select - '${field.fieldDef.name}'`);
                 }
@@ -1727,7 +1752,7 @@ class QueryQuery extends QueryField {
                 this.expandFields(fir);
                 resultStruct.add(as, fir);
             }
-            else if (field instanceof QueryAtomicField) {
+            else if (field instanceof AbstractQueryAtomic) {
                 resultStruct.addField(as, field, {
                     resultIndex,
                     type: 'result',
@@ -1738,16 +1763,12 @@ class QueryQuery extends QueryField {
                         throw new Error(`Aggregate Fields cannot be used in select - '${field.fieldDef.name}'`);
                     }
                 }
-                // } else if (field instanceof QueryStruct) {
-                //   // this could probably be optimized.  We are adding the primary key of the joined structure
-                //   //  instead of the foreignKey.  We have to do this in at least the INNER join case
-                //   //  so i'm just going to let the SQL database do the optimization (which is pretty rudimentary)
-                //   const pkFieldDef = field.getAsQueryField();
-                //   resultStruct.addField(as, pkFieldDef, {
-                //     resultIndex,
-                //     type: "result",
-                //   });
-                //   resultStruct.addStructToJoin(field, false);
+            }
+            else if (field instanceof QueryFieldStruct) {
+                resultStruct.addField(as, field, {
+                    resultIndex,
+                    type: 'result',
+                });
             }
             // else if (
             //   this.firstSegment.type === "project" &&
@@ -1818,8 +1839,8 @@ class QueryQuery extends QueryField {
             const alwaysJoins = (_a = stage.alwaysJoins) !== null && _a !== void 0 ? _a : [];
             for (const joinName of alwaysJoins) {
                 const qs = this.parent.getChildByName(joinName);
-                if (qs instanceof QueryStruct) {
-                    rootResult.addStructToJoin(qs, this, undefined, []);
+                if (qs instanceof QueryFieldStruct) {
+                    rootResult.addStructToJoin(qs.queryStruct, this, undefined, []);
                 }
             }
         }
@@ -1922,10 +1943,6 @@ class QueryQuery extends QueryField {
             }
             else if (fi instanceof FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
-                    // mtoy todo -- remember why you commented this out
-                    // if (fi.f instanceof QueryFieldStruct) {
-                    //   fields.push(fi.f.getAsJoinedStructDef(name));
-                    // }
                     // if there is only one dimension, it is the primaryKey
                     //  if there are more, primaryKey is undefined.
                     if (isScalarField(fi.f)) {
@@ -1937,16 +1954,25 @@ class QueryQuery extends QueryField {
                         }
                         dimCount++;
                     }
-                    const location = fi.f.fieldDef.location;
-                    const annotation = fi.f.fieldDef.annotation;
+                    // Remove computations because they are all resolved
+                    let fOut = fi.f.fieldDef;
+                    if ((0, malloy_types_1.hasExpression)(fOut)) {
+                        fOut = { ...fOut };
+                        // "as" because delete needs the property to be optional
+                        delete fOut.e;
+                        delete fOut.code;
+                        delete fOut.expressionType;
+                    }
+                    const location = fOut.location;
+                    const annotation = fOut.annotation;
                     // build out the result fields...
-                    switch (fi.f.fieldDef.type) {
+                    switch (fOut.type) {
                         case 'boolean':
                         case 'json':
                         case 'string':
                             fields.push({
                                 name,
-                                type: fi.f.fieldDef.type,
+                                type: fOut.type,
                                 resultMetadata,
                                 location,
                                 annotation,
@@ -1954,8 +1980,8 @@ class QueryQuery extends QueryField {
                             break;
                         case 'date':
                         case 'timestamp': {
-                            const timeframe = fi.f.fieldDef.timeframe;
-                            const fd = { type: fi.f.fieldDef.type };
+                            const timeframe = fOut.timeframe;
+                            const fd = { type: fOut.type };
                             if (timeframe) {
                                 fd.timeframe = timeframe;
                             }
@@ -1971,7 +1997,7 @@ class QueryQuery extends QueryField {
                         case 'number':
                             fields.push({
                                 name,
-                                numberType: fi.f.fieldDef.numberType,
+                                numberType: fOut.numberType,
                                 type: 'number',
                                 resultMetadata,
                                 location,
@@ -1979,10 +2005,13 @@ class QueryQuery extends QueryField {
                             });
                             break;
                         case 'sql native':
-                            fields.push({ ...fi.f.fieldDef, resultMetadata, location });
+                        case 'record':
+                        case 'array': {
+                            fields.push({ ...fOut, resultMetadata });
                             break;
+                        }
                         default:
-                            throw new Error(`unknown Field Type in query ${JSON.stringify(fi.f.fieldDef)}`);
+                            throw new Error(`unknown Field Type in query ${JSON.stringify(fOut)}`);
                     }
                 }
             }
@@ -2002,7 +2031,7 @@ class QueryQuery extends QueryField {
         }
         return outputStruct;
     }
-    generateSQLJoinBlock(stageWriter, ji) {
+    generateSQLJoinBlock(stageWriter, ji, depth) {
         var _a;
         let s = '';
         const qs = ji.queryStruct;
@@ -2063,7 +2092,7 @@ class QueryQuery extends QueryField {
                 let select = `SELECT ${ji.alias}.*`;
                 let joins = '';
                 for (const childJoin of ji.children) {
-                    joins += this.generateSQLJoinBlock(stageWriter, childJoin);
+                    joins += this.generateSQLJoinBlock(stageWriter, childJoin, depth + 1);
                     select += `, ${this.parent.dialect.sqlSelectAliasAsStruct(childJoin.alias, getDialectFieldList(childJoin.queryStruct.structDef))} AS ${childJoin.alias}`;
                 }
                 select += `\nFROM ${structSQL} AS ${ji.alias}\n${joins}\nWHERE ${conditions === null || conditions === void 0 ? void 0 : conditions.join(' AND ')}\n`;
@@ -2075,10 +2104,27 @@ class QueryQuery extends QueryField {
             if (qs.parent === undefined || ji.parent === undefined) {
                 throw new Error('Internal Error, nested structure with no parent.');
             }
-            const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), qsDef.name, 'struct', qs.parent.structDef.type === 'array', (0, malloy_types_1.isScalarArray)(this.parent.structDef));
+            // We need an SQL expression which results in the array for us to pass to un-nest
+            let arrayExpression;
+            if ((0, malloy_types_1.hasExpression)(qsDef)) {
+                // If this array is NOT contained in the parent, but a computed entity
+                // then the thing we are joining is not "parent.childName", but
+                // the expression which is built in that namespace
+                arrayExpression = this.exprToSQL(this.rootResult, qs.parent, qsDef.e);
+            }
+            else {
+                // If this is a reference through an expression at the top level,
+                // need to generate the expression because the expression is written
+                // in the top level, this call is being used to generate the join.
+                // Below the top level, the expression will have been written into
+                // a join at the top level, and the name will exist.
+                // ... not sure this is the right way to do this
+                // ... the test for this is called "source repeated record containing an array"
+                arrayExpression = qs.parent.sqlChildReference(qsDef.name, depth === 0 ? { result: this.rootResult, field: this } : undefined);
+            }
             // we need to generate primary key.  If parent has a primary key combine
             // console.log(ji.alias, fieldExpression, this.inNestedPipeline());
-            s += `${this.parent.dialect.sqlUnnestAlias(fieldExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, (0, malloy_types_1.isScalarArray)(qsDef), this.inNestedPipeline())}\n`;
+            s += `${this.parent.dialect.sqlUnnestAlias(arrayExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, (0, malloy_types_1.isScalarArray)(qsDef), this.inNestedPipeline())}\n`;
         }
         else if (qsDef.type === 'record') {
             throw new Error('Internal Error: records should never appear in join trees');
@@ -2087,7 +2133,7 @@ class QueryQuery extends QueryField {
             throw new Error(`Join type not implemented ${qs.structDef.type}`);
         }
         for (const childJoin of ji.children) {
-            s += this.generateSQLJoinBlock(stageWriter, childJoin);
+            s += this.generateSQLJoinBlock(stageWriter, childJoin, depth + 1);
         }
         return s;
     }
@@ -2134,7 +2180,7 @@ class QueryQuery extends QueryField {
             throw new Error('Internal Error, queries must start from a basetable');
         }
         for (const childJoin of ji.children) {
-            s += this.generateSQLJoinBlock(stageWriter, childJoin);
+            s += this.generateSQLJoinBlock(stageWriter, childJoin, 0);
         }
         return s;
     }
@@ -2716,9 +2762,10 @@ class QueryQuery extends QueryField {
             };
             pipeline.shift();
             for (const transform of pipeline) {
-                const s = new QueryStruct(structDef, undefined, {
-                    model: this.parent.getModel(),
-                }, this.parent.prepareResultOptions);
+                const parent = this.parent.parent
+                    ? { struct: this.parent.parent }
+                    : { model: this.parent.getModel() };
+                const s = new QueryStruct(structDef, undefined, parent, this.parent.prepareResultOptions);
                 const q = QueryQuery.makeQuery({ type: 'turtle', name: '~computeLastStage~', pipeline: [transform] }, s, stageWriter, this.isJoinedSubquery);
                 q.prepare(stageWriter);
                 lastStageName = q.generateSQL(stageWriter);
@@ -2761,7 +2808,7 @@ class QueryQueryIndexStage extends QueryQuery {
                 resultIndex,
                 type: 'result',
             });
-            if (field instanceof QueryAtomicField) {
+            if (field instanceof AbstractQueryAtomic) {
                 this.addDependancies(resultStruct, field);
             }
             resultIndex++;
@@ -2876,7 +2923,7 @@ class QueryQueryRaw extends QueryQuery {
     }
     getResultStructDef() {
         if (!(0, malloy_types_1.isSourceDef)(this.parent.structDef)) {
-            throw new Error(`Result cannot by type ${this.parent.structDef.type}`);
+            throw new Error(`Result cannot be type ${this.parent.structDef.type}`);
         }
         return { ...this.parent.structDef, type: 'query_result' };
     }
@@ -2906,10 +2953,9 @@ class QueryQueryIndex extends QueryQuery {
                 const stage = stageMap[stageRoot];
                 if (stage === undefined) {
                     const f = this.parent.nameMap.get(fref.path[0]);
-                    if (f instanceof QueryStruct &&
-                        (0, malloy_types_1.isJoined)(f.structDef) &&
-                        f.structDef.join === 'many' &&
-                        f.structDef.fields.length > 1) {
+                    if (f instanceof QueryFieldStruct &&
+                        f.fieldDef.join === 'many' &&
+                        f.fieldDef.fields.length > 1) {
                         const toStage = [fref];
                         stageMap[stageRoot] = toStage;
                         this.stages.push(toStage);
@@ -2978,18 +3024,39 @@ class QueryQueryIndex extends QueryQuery {
         return ret;
     }
 }
-/** Structure object as it is used to build a query */
 /*
-  Sometimes this is built from a field def, as in a join
-
-  But sometimes this is built as the intermediate stage between pipelines
-  and in that case it doesn't have a fieldDef which bugs me because querynode
-  always has a fielddef so i think that is wrong too
-
-*/
-class QueryStruct extends QueryNode {
-    constructor(structDef, sourceArguments, parent, prepareResultOptions, referenceId) {
-        super(referenceId);
+ * The input to a query will always be a QueryStruct. A QueryStruct is also a namespace
+ * for tracking joins, and so a QueryFieldStruct is a QueryField which has a QueryStruct.
+ *
+ * This is a result of it being impossible to inherit both from QueryStruct and QueryField
+ * for array and record types.
+ */
+class QueryFieldStruct extends QueryField {
+    constructor(jfd, sourceArguments, parent, prepareResultOptions, referenceId) {
+        super(jfd, parent, referenceId);
+        this.fieldDef = jfd;
+        this.queryStruct = new QueryStruct(jfd, sourceArguments, { struct: parent }, prepareResultOptions);
+    }
+    /*
+     * Proxy the field-like methods that QueryStruct implements, eventually
+     * those probably should be in here ... I thought this would be important
+     * but maybe it isn't, it doesn't fix the problem I am working on ...
+     */
+    // mtoy todo review with lloyd if any of these are needed, had to NOT
+    // do getIdentifier to pass a test, didn't stop to think why.
+    // getIdentifier() {
+    //   return this.queryStruct.getIdentifier();
+    // }
+    getJoinableParent() {
+        return this.queryStruct.getJoinableParent();
+    }
+    getFullOutputName() {
+        return this.queryStruct.getFullOutputName();
+    }
+}
+/** Structure object as it is used to build a query */
+class QueryStruct {
+    constructor(structDef, sourceArguments, parent, prepareResultOptions) {
         this.structDef = structDef;
         this.sourceArguments = sourceArguments;
         this.prepareResultOptions = prepareResultOptions;
@@ -3014,6 +3081,9 @@ class QueryStruct extends QueryNode {
         this.dialect = (0, dialect_1.getDialect)(structDef.dialect);
         this.addFieldsFromFieldList(structDef.fields);
     }
+    informOfAliasValue(av) {
+        this.recordAlias = av;
+    }
     maybeEmitParameterizedSourceUsage() {
         var _a;
         if ((0, malloy_types_1.isSourceDef)(this.structDef)) {
@@ -3076,15 +3146,15 @@ class QueryStruct extends QueryNode {
     addFieldsFromFieldList(fields) {
         for (const field of fields) {
             const as = (0, malloy_types_1.getIdentifier)(field);
-            if ((0, malloy_types_1.isJoined)(field)) {
-                this.addFieldToNameMap(as, new QueryStruct(field, undefined, { struct: this }, this.prepareResultOptions));
-            }
-            else if (field.type === 'turtle') {
+            if (field.type === 'turtle') {
                 this.addFieldToNameMap(as, QueryQuery.makeQuery(field, this, undefined, false));
             }
-            else {
+            else if ((0, malloy_types_1.isAtomic)(field) || (0, malloy_types_1.isJoinedSource)(field)) {
                 this.addFieldToNameMap(as, this.makeQueryField(field));
             }
+            else {
+                throw new Error('mtoy did nit add field');
+            }
         }
         // if we don't have distinct key yet for this struct, add it.
         if (!this.nameMap.has('__distinct_key')) {
@@ -3133,15 +3203,47 @@ class QueryStruct extends QueryNode {
             return this.getIdentifier();
         }
     }
+    sqlChildReference(name, expand) {
+        let parentRef = this.getSQLIdentifier();
+        if (expand && (0, malloy_types_1.isAtomic)(this.structDef) && (0, malloy_types_1.hasExpression)(this.structDef)) {
+            parentRef = expand.field.exprToSQL(expand.result, this, this.structDef.e);
+        }
+        let refType = 'table';
+        if (this.structDef.type === 'record') {
+            refType = 'record';
+        }
+        else if (this.structDef.type === 'array') {
+            refType =
+                this.structDef.elementTypeDef.type === 'record_element'
+                    ? 'array[record]'
+                    : 'array[scalar]';
+        }
+        else if (this.structDef.type === 'nest_source') {
+            refType = 'nest source';
+        }
+        const child = this.getChildByName(name);
+        const childType = (child === null || child === void 0 ? void 0 : child.fieldDef.type) || 'unknown';
+        return this.dialect.sqlFieldReference(parentRef, refType, name, childType);
+    }
     // return the name of the field in SQL
     getIdentifier() {
         // if it is the root table, use provided alias if we have one.
         if ((0, malloy_types_1.isBaseTable)(this.structDef)) {
             return 'base';
         }
+        // If this is a synthetic column, return the expression rather than the name
+        // because the name will not exist. Only for records because the other types
+        // will have joins and thus be in the namespace. We can't compute it here
+        // because we don't have access to the Query to call exprToSQL.
+        if (this.structDef.type === 'record' && (0, malloy_types_1.hasExpression)(this.structDef)) {
+            if (this.recordAlias) {
+                return this.recordAlias;
+            }
+            throw new Error('INTERNAL ERROR, record field alias not pre-computed');
+        }
         // if this is an inline object, include the parents alias.
         if (this.structDef.type === 'record' && this.parent) {
-            return (this.parent.getSQLIdentifier() + '.' + (0, malloy_types_1.getIdentifier)(this.structDef));
+            return this.parent.sqlChildReference((0, malloy_types_1.getIdentifier)(this.structDef), undefined);
         }
         // we are somewhere in the join tree.  Make sure the alias is unique.
         return this.getAliasIdentifier();
@@ -3222,8 +3324,8 @@ class QueryStruct extends QueryNode {
             }
         }
         for (const [, v] of this.nameMap) {
-            if (v instanceof QueryStruct) {
-                v.resolveQueryFields();
+            if (v instanceof QueryFieldStruct) {
+                v.queryStruct.resolveQueryFields();
             }
         }
     }
@@ -3255,6 +3357,13 @@ class QueryStruct extends QueryNode {
     /** makes a new queryable field object from a fieldDef */
     makeQueryField(field, referenceId) {
         switch (field.type) {
+            case 'array':
+            case 'record':
+            case 'query_source':
+            case 'table':
+            case 'sql_select':
+            case 'composite':
+                return new QueryFieldStruct(field, undefined, this, this.prepareResultOptions);
             case 'string':
                 return new QueryFieldString(field, this, referenceId);
             case 'date':
@@ -3269,13 +3378,10 @@ class QueryStruct extends QueryNode {
                 return new QueryFieldJSON(field, this, referenceId);
             case 'sql native':
                 return new QueryFieldUnsupported(field, this, referenceId);
-            // case "reduce":
-            // case "project":
-            // case "index":
             case 'turtle':
-                return new QueryTurtle(field, this, referenceId);
+                return QueryQuery.makeQuery(field, this, undefined, false);
             default:
-                throw new Error(`unknown field definition ${JSON.stringify(field)}`);
+                throw new Error(`unknown field definition ${(JSON.stringify(field), undefined, 2)}`);
         }
     }
     structSourceSQL(stageWriter) {
@@ -3323,35 +3429,43 @@ class QueryStruct extends QueryNode {
     }
     /** convert a path into a field reference */
     getFieldByName(path) {
-        return path.reduce((lookIn, childName) => {
-            const r = lookIn.getChildByName(childName);
-            if (r === undefined) {
-                throw new Error(path.length === 1
-                    ? `'${childName}' not found`
-                    : `'${childName}' not found in '${path.join('.')}'`);
+        let found = undefined;
+        let lookIn = this;
+        let notFound = path[0];
+        for (const n of path) {
+            found = lookIn === null || lookIn === void 0 ? void 0 : lookIn.getChildByName(n);
+            if (!found) {
+                notFound = n;
+                break;
             }
-            return r;
-        }, this);
+            lookIn =
+                found instanceof QueryFieldStruct ? found.queryStruct : undefined;
+        }
+        if (found === undefined) {
+            const pathErr = path.length > 1 ? ` in ${path.join('.')}` : '';
+            throw new Error(`${notFound} not found${pathErr}`);
+        }
+        return found;
     }
     // structs referenced in queries are converted to fields.
     getQueryFieldByName(name) {
         const field = this.getFieldByName(name);
-        if (field instanceof QueryStruct) {
+        if (field instanceof QueryFieldStruct) {
             throw new Error(`Cannot reference ${name.join('.')} as a scalar'`);
         }
         return field;
     }
-    getQueryFieldReference(name, annotation) {
-        const field = this.getFieldByName(name);
+    getQueryFieldReference(path, annotation) {
+        const field = this.getFieldByName(path);
         if (annotation) {
             if (field.parent === undefined) {
-                throw new Error('Unexpected reference to orphaned query field');
+                throw new Error('Inconcievable, field reference to orphaned query field');
             }
             // Made a field object from the source, but the annotations were computed by the compiler
             // when it generated the reference, and has both the source and reference annotations included.
-            if (field instanceof QueryStruct) {
-                const newDef = { ...field.structDef, annotation };
-                return new QueryStruct(newDef, undefined, field.parent, {}, field.referenceId);
+            if (field instanceof QueryFieldStruct) {
+                const newDef = { ...field.fieldDef, annotation };
+                return new QueryFieldStruct(newDef, undefined, field.parent, {}, field.referenceId);
             }
             else {
                 const newDef = { ...field.fieldDef, annotation };
@@ -3361,29 +3475,30 @@ class QueryStruct extends QueryNode {
         return field;
     }
     getDimensionOrMeasureByName(name) {
-        const query = this.getFieldByName(name);
-        if (query instanceof QueryAtomicField) {
-            return query;
+        const field = this.getFieldByName(name);
+        if (!field.isAtomic()) {
+            throw new Error(`${name} is not an atomic field? Inconceivable!`);
         }
-        throw new Error(`${name} is not an atomic field? Inconceivable!`);
+        return field;
     }
     /** returns a query object for the given name */
     getDimensionByName(name) {
-        const query = this.getFieldByName(name);
-        if (query instanceof QueryAtomicField && isScalarField(query)) {
-            return query;
+        const field = this.getFieldByName(name);
+        if (field.isAtomic() && isScalarField(field)) {
+            return field;
         }
         throw new Error(`${name} is not an atomic scalar field? Inconceivable!`);
     }
     /** returns a query object for the given name */
     getStructByName(name) {
-        const struct = this.getFieldByName(name);
-        if (struct instanceof QueryStruct) {
-            return struct;
+        if (name.length === 0) {
+            return this;
         }
-        else {
-            throw new Error(`Error: Path to structure not found '${name.join('.')}'`);
+        const struct = this.getFieldByName(name);
+        if (struct instanceof QueryFieldStruct) {
+            return struct.queryStruct;
         }
+        throw new Error(`Error: Path to structure not found '${name.join('.')}'`);
     }
     getDistinctKey() {
         if (this.structDef.type !== 'record') {
@@ -3532,7 +3647,7 @@ class QueryModel {
         const struct = this.getStructByName(explore);
         let indexStar = [];
         for (const [fn, fv] of struct.nameMap) {
-            if (!(fv instanceof QueryStruct)) {
+            if (!(fv instanceof QueryFieldStruct)) {
                 if (isScalarField(fv) && fv.includeInWildcard()) {
                     indexStar.push({ type: 'fieldref', path: [fn] });
                 }