@malloydata/malloy 0.0.195-dev241003204819 → 0.0.195-dev241007154000

package/dist/model/malloy_query.js

@@ -41,7 +41,7 @@ function identifierNormalize(s) {
 }
 function getDialectFieldList(structDef) {
     const dialectFieldList = [];
-    for (const f of structDef.fields.filter(malloy_types_1.isPhysical)) {
+    for (const f of structDef.fields.filter(malloy_types_1.fieldIsIntrinsic)) {
         dialectFieldList.push({
             type: f.type,
             sqlExpression: (0, malloy_types_1.getIdentifier)(f),
@@ -205,11 +205,8 @@ class GenerateState {
     }
 }
 class QueryNode {
-    constructor(fieldDef) {
-        this.fieldDef = fieldDef;
-    }
-    getIdentifier() {
-        return (0, malloy_types_1.getIdentifier)(this.fieldDef);
+    constructor(referenceId) {
+        this.referenceId = referenceId !== null && referenceId !== void 0 ? referenceId : (0, uuid_1.v4)();
     }
     getChildByName(_name) {
         return undefined;
@@ -217,24 +214,21 @@ class QueryNode {
 }
 class QueryField extends QueryNode {
     constructor(fieldDef, parent, referenceId) {
-        super(fieldDef);
+        super(referenceId);
+        this.fieldDef = fieldDef;
         this.parent = parent;
         this.fieldDef = fieldDef;
-        this.referenceId = referenceId !== null && referenceId !== void 0 ? referenceId : (0, uuid_1.v4)();
+    }
+    getIdentifier() {
+        return (0, malloy_types_1.getIdentifier)(this.fieldDef);
     }
     uniqueKeyPossibleUse() {
         return undefined;
     }
     getJoinableParent() {
-        // if it is inline it should always have a parent
         const parent = this.parent;
-        if (parent.fieldDef.structRelationship.type === 'inline') {
-            if (parent) {
-                return parent.getJoinableParent();
-            }
-            else {
-                throw new Error('Internal Error: inline struct cannot be root');
-            }
+        if (parent.structDef.type === 'record') {
+            return parent.getJoinableParent();
         }
         return parent;
     }
@@ -876,19 +870,30 @@ class QueryField extends QueryNode {
                 throw new Error(`Internal Error: Unknown expression node '${expr.node}' ${JSON.stringify(expr, undefined, 2)}`);
         }
     }
+    isNestedInParent(parentDef) {
+        switch (parentDef.type) {
+            case 'record':
+            case 'array':
+                return true;
+                return true;
+            default:
+                return false;
+        }
+    }
+    isArrayElement(parentDef) {
+        return (parentDef.type === 'array' &&
+            parentDef.elementTypeDef.type !== 'record_element');
+    }
     getExpr() {
         if ((0, malloy_types_1.hasExpression)(this.fieldDef)) {
             return this.fieldDef.e;
         }
+        const pType = this.parent.structDef.type;
         return {
             node: 'genericSQLExpr',
             kids: { args: [] },
             src: [
-                this.parent.dialect.sqlFieldReference(this.parent.getSQLIdentifier(), this.fieldDef.name, this.fieldDef.type, this.parent.fieldDef.structSource.type === 'nested' ||
-                    this.parent.fieldDef.structSource.type === 'inline' ||
-                    (this.parent.fieldDef.structSource.type === 'sql' &&
-                        this.parent.fieldDef.structSource.method === 'nested'), this.parent.fieldDef.structRelationship.type === 'nested' &&
-                    this.parent.fieldDef.structRelationship.isArray),
+                this.parent.dialect.sqlFieldReference(this.parent.getSQLIdentifier(), this.fieldDef.name, this.fieldDef.type, pType === 'record' || pType === 'array' || pType === 'nest_source', (0, malloy_types_1.isScalarArray)(this.parent.structDef)),
             ],
         };
     }
@@ -906,14 +911,20 @@ function isScalarField(f) {
     return f instanceof QueryAtomicField && !f.isCalculated() && !f.isAggregate();
 }
 class QueryAtomicField extends QueryField {
+    constructor(fieldDef, parent, refId) {
+        super(fieldDef, parent, refId);
+        this.fieldDef = fieldDef; // wish I didn't have to do this
+    }
     includeInWildcard() {
         return true;
     }
     isCalculated() {
-        return (0, malloy_types_1.expressionIsCalculation)(this.fieldDef.expressionType);
+        return ((0, malloy_types_1.hasExpression)(this.fieldDef) &&
+            (0, malloy_types_1.expressionIsCalculation)(this.fieldDef.expressionType));
     }
     isAggregate() {
-        return (0, malloy_types_1.expressionIsAggregate)(this.fieldDef.expressionType);
+        return ((0, malloy_types_1.hasExpression)(this.fieldDef) &&
+            (0, malloy_types_1.expressionIsAggregate)(this.fieldDef.expressionType));
     }
     getFilterList() {
         return [];
@@ -933,35 +944,6 @@ class QueryFieldJSON extends QueryAtomicField {
 }
 class QueryFieldUnsupported extends QueryAtomicField {
 }
-// in a query a struct can be referenced.  The struct will
-//  emit the primary key field in the actual result set and
-//  will include the StructDef as a foreign key join in the output
-//  StructDef.
-class QueryFieldStruct extends QueryAtomicField {
-    constructor(fieldDef, parent, primaryKey) {
-        super(fieldDef, parent);
-        this.primaryKey = primaryKey;
-    }
-    getName() {
-        return (0, malloy_types_1.getIdentifier)(this.fieldDef);
-    }
-    getAsJoinedStructDef(foreignKeyName) {
-        return {
-            ...this.parent.fieldDef,
-            structRelationship: {
-                type: 'one',
-                matrixOperation: 'left',
-                onExpression: {
-                    node: '=',
-                    kids: {
-                        left: { node: 'field', path: [this.primaryKey] },
-                        right: { node: 'field', path: [foreignKeyName] },
-                    },
-                },
-            },
-        };
-    }
-}
 class QueryFieldDate extends QueryAtomicField {
     generateExpression(resultSet) {
         const fd = this.fieldDef;
@@ -1005,14 +987,14 @@ class QueryFieldDistinctKey extends QueryAtomicField {
             const pk = this.parent.getPrimaryKeyField(this.fieldDef);
             return pk.generateExpression(resultSet);
         }
-        else if (this.parent.fieldDef.structSource.type === 'nested') {
+        else if (this.parent.structDef.type === 'array') {
             const parentKey = (_a = this.parent.parent) === null || _a === void 0 ? void 0 : _a.getDistinctKey().generateExpression(resultSet);
             return this.parent.dialect.sqlMakeUnnestKey(parentKey || '', // shouldn't have to do this...
             this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), '__row_id', 'string', true, false));
         }
         else {
             // return this.parent.getIdentifier() + "." + "__distinct_key";
-            return this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), '__distinct_key', 'string', this.parent.fieldDef.structRelationship.type === 'nested', false);
+            return this.parent.dialect.sqlFieldReference(this.parent.getIdentifier(), '__distinct_key', 'string', false, false);
         }
     }
     includeInWildcard() {
@@ -1257,7 +1239,7 @@ class FieldInstanceResult {
             if (fi instanceof FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
                     if (fi.f.fieldDef.type === 'turtle' ||
-                        fi.f.fieldDef.type === 'struct' ||
+                        (0, malloy_types_1.isJoined)(fi.f.fieldDef) ||
                         (0, malloy_types_1.expressionIsAnalytic)(fi.f.fieldDef.expressionType)) {
                         continue;
                     }
@@ -1297,12 +1279,12 @@ class FieldInstanceResult {
             parent = this.root().joins.get(parentStruct.getIdentifier());
         }
         // add any dependant joins based on the ON
-        const sr = qs.fieldDef.structRelationship;
-        if ((0, malloy_types_1.isJoinOn)(sr) &&
+        const sd = qs.structDef;
+        if ((0, malloy_types_1.isJoinedSource)(sd) &&
             qs.parent && // if the join has an ON, it must thave a parent
-            sr.onExpression !== undefined &&
+            sd.onExpression &&
             joinStack.indexOf(name) === -1) {
-            query.addDependantExpr(this, qs.parent, sr.onExpression, [
+            query.addDependantExpr(this, qs.parent, sd.onExpression, [
                 ...joinStack,
                 name,
             ]);
@@ -1457,7 +1439,7 @@ class FieldInstanceResultRoot extends FieldInstanceResult {
                     if (!j.queryStruct.primaryKey()) {
                         j.makeUniqueKey = true;
                     }
-                    if (j.queryStruct.fieldDef.structRelationship.type === 'nested') {
+                    if (j.queryStruct.structDef.type === 'array') {
                         j = j.parent;
                     }
                     else {
@@ -1482,54 +1464,50 @@ class JoinInstance {
         }
         // convert the filter list into a list of boolean fields so we can
         //  generate dependancies and code for them.
-        if (this.queryStruct.fieldDef.filterList) {
-            this.joinFilterConditions = [];
-            for (const filter of this.queryStruct.fieldDef.filterList) {
-                const qf = new QueryFieldBoolean({
-                    type: 'boolean',
-                    name: 'ignoreme',
-                    e: filter.e,
-                }, this.queryStruct);
-                this.joinFilterConditions.push(qf);
-            }
+        const sd = this.queryStruct.structDef;
+        if ((0, malloy_types_1.isSourceDef)(sd) && sd.filterList) {
+            this.joinFilterConditions = sd.filterList.map(filter => new QueryFieldBoolean({
+                type: 'boolean',
+                name: 'ignoreme',
+                e: filter.e,
+            }, this.queryStruct));
         }
     }
     parentRelationship() {
         if (this.queryStruct.parent === undefined) {
             return 'root';
         }
-        switch (this.queryStruct.fieldDef.structRelationship.type) {
-            case 'one':
-                return 'many_to_one';
-            case 'cross':
-                return 'many_to_many';
-            case 'many':
-                return 'one_to_many';
-            case 'nested':
-                return 'one_to_many';
-            case 'inline':
-                return 'one_to_one';
-            default:
-                throw new Error(`Internal error unknown relationship type to parent for ${this.queryStruct.fieldDef.name}`);
+        const thisStruct = this.queryStruct.structDef;
+        if ((0, malloy_types_1.isJoined)(thisStruct)) {
+            switch (thisStruct.join) {
+                case 'one':
+                    return 'many_to_one';
+                case 'cross':
+                    return 'many_to_many';
+                case 'many':
+                    return 'one_to_many';
+            }
         }
+        throw new Error(`Internal error unknown relationship type to parent for ${this.queryStruct.structDef.name}`);
     }
     // For now, we force all symmetric calculations for full and right joins
     //  because we need distinct keys for COUNT(xx) operations.  Don't really need
     //  this for sums.  This will produce correct results and we can optimize this
     //  at some point..
     forceAllSymmetricCalculations() {
-        const sr = this.queryStruct.fieldDef.structRelationship;
-        if (this.queryStruct.parent === undefined || !(0, malloy_types_1.isJoinOn)(sr)) {
+        if (this.queryStruct.parent === undefined) {
             return false;
         }
-        if (sr.matrixOperation === 'right' || sr.matrixOperation === 'full') {
-            return true;
+        const thisStruct = this.queryStruct.structDef;
+        if ((0, malloy_types_1.isJoined)(thisStruct)) {
+            return (thisStruct.matrixOperation === 'right' ||
+                thisStruct.matrixOperation === 'full');
         }
         return false;
     }
     // postgres unnest needs to know the names of the physical fields.
     getDialectFieldList() {
-        return getDialectFieldList(this.queryStruct.fieldDef);
+        return getDialectFieldList(this.queryStruct.structDef);
     }
 }
 /** nested query */
@@ -1544,7 +1522,7 @@ class Segment {
     static nextStructDef(structDef, segment) {
         const qs = new QueryStruct(structDef, undefined, {
             model: new QueryModel(undefined),
-        });
+        }, {});
         const turtleDef = {
             type: 'turtle',
             name: 'ignoreme',
@@ -1574,7 +1552,7 @@ class QueryQuery extends QueryField {
         let parent = parentStruct;
         let turtleWithFilters = parentStruct.applyStructFiltersToTurtleDef(fieldDef);
         const firstStage = turtleWithFilters.pipeline[0];
-        const sourceDef = parentStruct.fieldDef;
+        const sourceDef = parentStruct.structDef;
         // if we are generating code
         //  and have extended declaration, we need to make a new QueryStruct
         //  copy the definitions into a new structdef
@@ -1597,7 +1575,8 @@ class QueryQuery extends QueryField {
                 ],
             };
         }
-        if (sourceDef.queryTimezone &&
+        if ((0, malloy_types_1.isSourceDef)(sourceDef) &&
+            sourceDef.queryTimezone &&
             (0, malloy_types_1.isQuerySegment)(firstStage) &&
             firstStage.queryTimezone === undefined) {
             firstStage.queryTimezone = sourceDef.queryTimezone;
@@ -1616,8 +1595,7 @@ class QueryQuery extends QueryField {
         }
     }
     inNestedPipeline() {
-        return (this.parent.fieldDef.structSource.type === 'sql' &&
-            this.parent.fieldDef.structSource.method === 'nested');
+        return this.parent.structDef.type === 'nest_source';
     }
     // get a field ref and expand it.
     expandField(f) {
@@ -1820,11 +1798,14 @@ class QueryQuery extends QueryField {
     getResultMetadata(fi) {
         if (fi instanceof FieldInstanceField) {
             if (fi.fieldUsage.type === 'result') {
+                // const fieldDef = fi.f.fieldDef as AtomicField;
                 const fieldDef = fi.f.fieldDef;
                 let filterList;
                 const sourceField = fi.f.parent.getFullOutputName() +
                     (fieldDef.name || fieldDef.as || 'undefined');
-                const sourceExpression = fieldDef.code;
+                const sourceExpression = (0, malloy_types_1.hasExpression)(fieldDef)
+                    ? fieldDef.code
+                    : undefined;
                 const sourceClasses = [sourceField];
                 const referenceId = fi.f.referenceId;
                 const base = {
@@ -1887,24 +1868,34 @@ class QueryQuery extends QueryField {
             const resultMetadata = this.getResultMetadata(fi);
             if (fi instanceof FieldInstanceResult) {
                 const { structDef } = this.generateTurtlePipelineSQL(fi, new StageWriter(true, undefined), '<nosource>');
-                // LTNOTE: This is probably broken now.  Need to look at the last stage
-                //  to figure out the resulting nested/inline state...
-                const resultType = fi.getRepeatedResultType() === 'nested' ? 'nested' : 'inline';
-                structDef.name = name;
-                structDef.structRelationship = {
-                    fieldName: name,
-                    type: resultType,
-                    isArray: false,
-                };
-                structDef.structSource = { type: resultType };
-                structDef.resultMetadata = resultMetadata;
-                fields.push(structDef);
+                if (fi.getRepeatedResultType() === 'nested') {
+                    const multiLineNest = {
+                        ...structDef,
+                        type: 'array',
+                        elementTypeDef: { type: 'record_element' },
+                        join: 'many',
+                        name,
+                        resultMetadata,
+                    };
+                    fields.push(multiLineNest);
+                }
+                else {
+                    const oneLineNest = {
+                        ...structDef,
+                        type: 'record',
+                        join: 'one',
+                        name,
+                        resultMetadata,
+                    };
+                    fields.push(oneLineNest);
+                }
             }
             else if (fi instanceof FieldInstanceField) {
                 if (fi.fieldUsage.type === 'result') {
-                    if (fi.f instanceof QueryFieldStruct) {
-                        fields.push(fi.f.getAsJoinedStructDef(name));
-                    }
+                    // mtoy todo -- remember why you commented this out
+                    // if (fi.f instanceof QueryFieldStruct) {
+                    //   fields.push(fi.f.getAsJoinedStructDef(name));
+                    // }
                     // if there is only one dimension, it is the primaryKey
                     //  if there are more, primaryKey is undefined.
                     if (isScalarField(fi.f)) {
@@ -1931,6 +1922,7 @@ class QueryQuery extends QueryField {
                                 annotation,
                             });
                             break;
+                        case 'date':
                         case 'timestamp': {
                             const timeframe = fi.f.fieldDef.timeframe;
                             if (timeframe) {
@@ -1954,17 +1946,6 @@ class QueryQuery extends QueryField {
                             }
                             break;
                         }
-                        case 'date': {
-                            fields.push({
-                                name,
-                                type: fi.f.fieldDef.type,
-                                timeframe: fi.f.fieldDef.timeframe,
-                                resultMetadata,
-                                location,
-                                annotation,
-                            });
-                            break;
-                        }
                         case 'number':
                             fields.push({
                                 name,
@@ -1985,21 +1966,17 @@ class QueryQuery extends QueryField {
             }
         }
         const outputStruct = {
-            fields,
+            type: 'query_result',
             name: this.resultStage || 'result',
-            dialect: this.parent.dialect.name,
+            fields,
+            dialect: this.parent.structDef.dialect,
             primaryKey,
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.parent.connectionName,
-            },
-            structSource: { type: 'query_result' },
+            connection: this.parent.connectionName,
             resultMetadata: this.getResultMetadata(this.rootResult),
-            type: 'struct',
             queryTimezone: resultStruct.getQueryInfo().queryTimezone,
         };
-        if (this.parent.fieldDef.modelAnnotation) {
-            outputStruct.modelAnnotation = this.parent.fieldDef.modelAnnotation;
+        if (this.parent.structDef.modelAnnotation) {
+            outputStruct.modelAnnotation = this.parent.structDef.modelAnnotation;
         }
         return outputStruct;
     }
@@ -2007,14 +1984,12 @@ class QueryQuery extends QueryField {
         var _a;
         let s = '';
         const qs = ji.queryStruct;
-        (_a = qs.eventStream) === null || _a === void 0 ? void 0 : _a.emit('join-used', {
-            name: qs.fieldDef.as,
-        });
+        const qsDef = qs.structDef;
+        (_a = qs.eventStream) === null || _a === void 0 ? void 0 : _a.emit('join-used', { name: (0, malloy_types_1.getIdentifier)(qsDef) });
         qs.maybeEmitParameterizedSourceUsage();
-        const structRelationship = qs.fieldDef.structRelationship;
-        let structSQL = qs.structSourceSQL(stageWriter);
-        if ((0, malloy_types_1.isJoinOn)(structRelationship)) {
-            const matrixOperation = structRelationship.matrixOperation.toUpperCase();
+        if ((0, malloy_types_1.isJoinedSource)(qsDef)) {
+            let structSQL = qs.structSourceSQL(stageWriter);
+            const matrixOperation = (qsDef.matrixOperation || 'left').toUpperCase();
             if (ji.makeUniqueKey) {
                 const passKeys = this.generateSQLPassthroughKeys(qs);
                 structSQL = `(SELECT ${qs.dialect.sqlGenerateUUID()} as ${qs.dialect.sqlMaybeQuoteIdentifier('__distinct_key')}, x.* ${passKeys} FROM ${structSQL} as x)`;
@@ -2023,11 +1998,11 @@ class QueryQuery extends QueryField {
             if (qs.parent === undefined) {
                 throw new Error('Expected joined struct to have a parent.');
             }
-            if (structRelationship.onExpression) {
+            if (qsDef.onExpression) {
                 onCondition = new QueryFieldBoolean({
                     type: 'boolean',
                     name: 'ignoreme',
-                    e: structRelationship.onExpression,
+                    e: qsDef.onExpression,
                 }, qs.parent).generateExpression(this.rootResult);
             }
             else {
@@ -2064,28 +2039,27 @@ class QueryQuery extends QueryField {
                 let joins = '';
                 for (const childJoin of ji.children) {
                     joins += this.generateSQLJoinBlock(stageWriter, childJoin);
-                    select += `, ${this.parent.dialect.sqlSelectAliasAsStruct(childJoin.alias, getDialectFieldList(childJoin.queryStruct.fieldDef))} AS ${childJoin.alias}`;
+                    select += `, ${this.parent.dialect.sqlSelectAliasAsStruct(childJoin.alias, getDialectFieldList(childJoin.queryStruct.structDef))} AS ${childJoin.alias}`;
                 }
                 select += `\nFROM ${structSQL} AS ${ji.alias}\n${joins}\nWHERE ${conditions === null || conditions === void 0 ? void 0 : conditions.join(' AND ')}\n`;
                 s += `${matrixOperation} JOIN (\n${(0, utils_1.indent)(select)}) AS ${ji.alias}\n  ON ${onCondition}\n`;
                 return s;
             }
         }
-        else if (structRelationship.type === 'nested') {
+        else if (qsDef.type === 'array') {
             if (qs.parent === undefined || ji.parent === undefined) {
                 throw new Error('Internal Error, nested structure with no parent.');
             }
-            const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), structRelationship.fieldName, 'struct', qs.parent.fieldDef.structRelationship.type === 'nested', this.parent.fieldDef.structRelationship.type === 'nested' &&
-                this.parent.fieldDef.structRelationship.isArray);
+            const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), qsDef.name, 'struct', qs.parent.structDef.type === 'array', (0, malloy_types_1.isScalarArray)(this.parent.structDef));
             // we need to generate primary key.  If parent has a primary key combine
             // console.log(ji.alias, fieldExpression, this.inNestedPipeline());
-            s += `${this.parent.dialect.sqlUnnestAlias(fieldExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, structRelationship.isArray, this.inNestedPipeline())}\n`;
+            s += `${this.parent.dialect.sqlUnnestAlias(fieldExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, (0, malloy_types_1.isScalarArray)(qsDef), this.inNestedPipeline())}\n`;
         }
-        else if (structRelationship.type === 'inline') {
-            throw new Error('Internal Error: inline structs should never appear in join trees');
+        else if (qsDef.type === 'record') {
+            throw new Error('Internal Error: records should never appear in join trees');
         }
         else {
-            throw new Error(`Join type not implemented ${JSON.stringify(qs.fieldDef.structRelationship)}`);
+            throw new Error(`Join type not implemented ${qs.structDef.type}`);
         }
         for (const childJoin of ji.children) {
             s += this.generateSQLJoinBlock(stageWriter, childJoin);
@@ -2124,8 +2098,7 @@ class QueryQuery extends QueryField {
                 structSQL = stageWriter.addStage(`SELECT * from ${structSQL} as x limit 100000 `);
             }
         }
-        const structRelationship = qs.fieldDef.structRelationship;
-        if (structRelationship.type === 'basetable') {
+        if ((0, malloy_types_1.isBaseTable)(qs.structDef)) {
             if (ji.makeUniqueKey) {
                 const passKeys = this.generateSQLPassthroughKeys(qs);
                 structSQL = `(SELECT ${qs.dialect.sqlGenerateUUID()} as ${qs.dialect.sqlMaybeQuoteIdentifier('__distinct_key')}, x.* ${passKeys} FROM ${structSQL} as x)`;
@@ -2460,7 +2433,7 @@ class QueryQuery extends QueryField {
             for (const m of output.groupsAggregated) {
                 output.sql[0] += `WHEN group_set=${m.fromGroup} THEN ${m.toGroup} `;
             }
-            output.sql[0] += ' ELSE group_set END as group_set';
+            output.sql[0] += 'ELSE group_set END as group_set';
         }
     }
     generateSQLDepthN(depth, stageWriter, stageName) {
@@ -2658,11 +2631,15 @@ class QueryQuery extends QueryField {
                 name: 'starthere',
                 pipeline,
             };
-            structDef.name = this.parent.dialect.sqlUnnestPipelineHead(repeatedResultType === 'inline_all_numbers', sourceSQLExpression);
-            structDef.structSource = { type: 'sql', method: 'nested' };
-            const qs = new QueryStruct(structDef, undefined, {
-                model: this.parent.getModel(),
-            }, this.parent.prepareResultOptions);
+            const inputStruct = {
+                type: 'nest_source',
+                name: '~pipe~',
+                pipeSQL: this.parent.dialect.sqlUnnestPipelineHead(repeatedResultType === 'inline_all_numbers', sourceSQLExpression),
+                fields: structDef.fields,
+                connection: structDef.connection,
+                dialect: structDef.dialect,
+            };
+            const qs = new QueryStruct(inputStruct, undefined, { model: this.parent.getModel() }, this.parent.prepareResultOptions);
             const q = QueryQuery.makeQuery(newTurtle, qs, stageWriter, this.isJoinedSubquery);
             pipeOut = q.generateSQLFromPipeline(stageWriter);
             // console.log(stageWriter.generateSQLStages());
@@ -2700,11 +2677,6 @@ class QueryQuery extends QueryField {
             return this.generateSimpleSQL(stageWriter);
         }
     }
-    toMalloy() {
-        let ret = `EXPLORE ${(0, malloy_types_1.getIdentifier)(this.parent.fieldDef)} | `;
-        ret += this.fieldDef.type.toUpperCase() + ' ';
-        return ret;
-    }
     generateSQLFromPipeline(stageWriter) {
         this.parent.maybeEmitParameterizedSourceUsage();
         this.prepare(stageWriter);
@@ -2715,20 +2687,20 @@ class QueryQuery extends QueryField {
             const pipeline = [...this.fieldDef.pipeline];
             let structDef = {
                 ...outputStruct,
-                structSource: { type: 'sql', method: 'lastStage' },
+                type: 'finalize',
             };
             pipeline.shift();
             for (const transform of pipeline) {
                 const s = new QueryStruct(structDef, undefined, {
                     model: this.parent.getModel(),
                 }, this.parent.prepareResultOptions);
-                const q = QueryQuery.makeQuery({ type: 'turtle', name: 'ignoreme', pipeline: [transform] }, s, stageWriter, this.isJoinedSubquery);
+                const q = QueryQuery.makeQuery({ type: 'turtle', name: '~computeLastStage~', pipeline: [transform] }, s, stageWriter, this.isJoinedSubquery);
                 q.prepare(stageWriter);
                 lastStageName = q.generateSQL(stageWriter);
                 outputStruct = q.getResultStructDef();
                 structDef = {
                     ...outputStruct,
-                    structSource: { type: 'sql', method: 'lastStage' },
+                    type: 'finalize',
                 };
             }
         }
@@ -2869,18 +2841,19 @@ FROM ${resultStage}\n`);
 }
 class QueryQueryRaw extends QueryQuery {
     generateSQL(stageWriter) {
-        const ssrc = this.parent.fieldDef.structSource;
-        if (ssrc.type !== 'sql' || ssrc.method !== 'subquery') {
+        if (this.parent.structDef.type !== 'sql_select') {
             throw new Error('Invalid struct for QueryQueryRaw, currently only supports SQL');
         }
-        const s = ssrc.sqlBlock.selectStr;
-        return stageWriter.addStage(s);
+        return stageWriter.addStage(this.parent.structDef.selectStr);
     }
     prepare() {
         // Do nothing!
     }
     getResultStructDef() {
-        return this.parent.fieldDef;
+        if (!(0, malloy_types_1.isSourceDef)(this.parent.structDef)) {
+            throw new Error(`Result cannot by type ${this.parent.structDef.type}`);
+        }
+        return this.parent.structDef;
     }
     getResultMetadata(_fi) {
         return undefined;
@@ -2909,9 +2882,9 @@ class QueryQueryIndex extends QueryQuery {
                 if (stage === undefined) {
                     const f = this.parent.nameMap.get(fref.path[0]);
                     if (f instanceof QueryStruct &&
-                        (f.fieldDef.structRelationship.type === 'many' ||
-                            f.fieldDef.structRelationship.type === 'nested') &&
-                        f.fieldDef.fields.length > 1) {
+                        (0, malloy_types_1.isJoined)(f.structDef) &&
+                        f.structDef.join === 'many' &&
+                        f.structDef.fields.length > 1) {
                         const toStage = [fref];
                         stageMap[stageRoot] = toStage;
                         this.stages.push(toStage);
@@ -2962,9 +2935,9 @@ class QueryQueryIndex extends QueryQuery {
      */
     getResultStructDef() {
         const ret = {
-            type: 'struct',
+            type: 'query_result',
             name: this.resultStage || 'result',
-            dialect: this.parent.fieldDef.dialect,
+            dialect: this.parent.structDef.dialect,
             fields: [
                 { type: 'string', name: 'fieldName' },
                 { type: 'string', name: 'fieldPath' },
@@ -2972,33 +2945,37 @@ class QueryQueryIndex extends QueryQuery {
                 { type: 'string', name: 'fieldType' },
                 { type: 'number', name: 'weight', numberType: 'integer' },
             ],
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.parent.connectionName,
-            },
-            structSource: { type: 'query_result' },
+            connection: this.parent.connectionName,
         };
-        if (this.parent.fieldDef.modelAnnotation) {
-            ret.modelAnnotation = this.parent.fieldDef.modelAnnotation;
+        if (this.parent.structDef.modelAnnotation) {
+            ret.modelAnnotation = this.parent.structDef.modelAnnotation;
         }
         return ret;
     }
 }
 /** Structure object as it is used to build a query */
+/*
+  Sometimes this is built from a field def, as in a join
+
+  But sometimes this is built as the intermediate stage between pipelines
+  and in that case it doesn't have a fieldDef which bugs me because querynode
+  always has a fielddef so i think that is wrong too
+
+*/
 class QueryStruct extends QueryNode {
-    constructor(fieldDef, sourceArguments, parent, prepareResultOptions) {
-        super(fieldDef);
+    constructor(structDef, sourceArguments, parent, prepareResultOptions, referenceId) {
+        super(referenceId);
+        this.structDef = structDef;
         this.sourceArguments = sourceArguments;
         this.prepareResultOptions = prepareResultOptions;
         this.nameMap = new Map();
         this._arguments = undefined;
-        this.stacki = new Error('').stack;
         this.setParent(parent);
         if ('model' in parent) {
             this.model = parent.model;
             this.pathAliasMap = new Map();
-            if (fieldDef.structRelationship.type === 'basetable') {
-                this.connectionName = fieldDef.structRelationship.connectionName;
+            if ((0, malloy_types_1.isSourceDef)(structDef)) {
+                this.connectionName = structDef.connection;
             }
             else {
                 throw new Error('All root StructDefs should be a baseTable');
@@ -3009,22 +2986,22 @@ class QueryStruct extends QueryNode {
             this.pathAliasMap = this.root().pathAliasMap;
             this.connectionName = this.root().connectionName;
         }
-        this.fieldDef = fieldDef; // shouldn't have to do this, but
-        // type script is missing a beat here.
-        this.dialect = (0, dialect_1.getDialect)(this.fieldDef.dialect);
-        this.addFieldsFromFieldList(this.fieldDef.fields);
+        this.dialect = (0, dialect_1.getDialect)(structDef.dialect);
+        this.addFieldsFromFieldList(structDef.fields);
     }
     maybeEmitParameterizedSourceUsage() {
         var _a;
-        const paramsAndArgs = {
-            ...this.fieldDef.parameters,
-            ...this.fieldDef.arguments,
-        };
-        if (Object.values(paramsAndArgs).length === 0)
-            return;
-        (_a = this.eventStream) === null || _a === void 0 ? void 0 : _a.emit('parameterized-source-compiled', {
-            parameters: paramsAndArgs,
-        });
+        if ((0, malloy_types_1.isSourceDef)(this.structDef)) {
+            const paramsAndArgs = {
+                ...this.structDef.parameters,
+                ...this.structDef.arguments,
+            };
+            if (Object.values(paramsAndArgs).length === 0)
+                return;
+            (_a = this.eventStream) === null || _a === void 0 ? void 0 : _a.emit('parameterized-source-compiled', {
+                parameters: paramsAndArgs,
+            });
+        }
     }
     resolveParentParameterReferences(param) {
         return {
@@ -3055,39 +3032,33 @@ class QueryStruct extends QueryNode {
             return this._arguments;
         }
         this._arguments = {};
-        // First, copy over all parameters, to get default values
-        const params = (_a = this.fieldDef.parameters) !== null && _a !== void 0 ? _a : {};
-        for (const parameterName in params) {
-            this._arguments[parameterName] = params[parameterName];
-        }
-        // Then, copy over arguments to override default values
-        const args = { ...this.fieldDef.arguments, ...this.sourceArguments };
-        for (const parameterName in args) {
-            const orig = args[parameterName];
-            this._arguments[parameterName] =
-                this.resolveParentParameterReferences(orig);
+        if ((0, malloy_types_1.isSourceDef)(this.structDef)) {
+            // First, copy over all parameters, to get default values
+            const params = (_a = this.structDef.parameters) !== null && _a !== void 0 ? _a : {};
+            for (const parameterName in params) {
+                this._arguments[parameterName] = params[parameterName];
+            }
+            // Then, copy over arguments to override default values
+            const args = { ...this.structDef.arguments, ...this.sourceArguments };
+            for (const parameterName in args) {
+                const orig = args[parameterName];
+                this._arguments[parameterName] =
+                    this.resolveParentParameterReferences(orig);
+            }
         }
         return this._arguments;
     }
     addFieldsFromFieldList(fields) {
         for (const field of fields) {
             const as = (0, malloy_types_1.getIdentifier)(field);
-            switch (field.type) {
-                case 'struct': {
-                    this.addFieldToNameMap(as, new QueryStruct(field, undefined, {
-                        struct: this,
-                    }, this.prepareResultOptions));
-                    break;
-                }
-                // case "reduce" || "project" || "index": {
-                case 'turtle': {
-                    // not sure why we need to cast here...
-                    this.addFieldToNameMap(as, QueryQuery.makeQuery(field, this, undefined, false));
-                    break;
-                }
-                default: {
-                    this.addFieldToNameMap(as, this.makeQueryField(field));
-                }
+            if ((0, malloy_types_1.isJoined)(field)) {
+                this.addFieldToNameMap(as, new QueryStruct(field, undefined, { struct: this }, this.prepareResultOptions));
+            }
+            else if (field.type === 'turtle') {
+                this.addFieldToNameMap(as, QueryQuery.makeQuery(field, this, undefined, false));
+            }
+            else {
+                this.addFieldToNameMap(as, this.makeQueryField(field));
             }
         }
         // if we don't have distinct key yet for this struct, add it.
@@ -3104,7 +3075,7 @@ class QueryStruct extends QueryNode {
         // make a unique alias name
         if (ret === undefined) {
             const aliases = Array.from(this.pathAliasMap.values());
-            const base = identifierNormalize((0, malloy_types_1.getIdentifier)(this.fieldDef));
+            const base = identifierNormalize((0, malloy_types_1.getIdentifier)(this.structDef));
             let name = `${base}_0`;
             let n = 1;
             while (aliases.includes(name) && n < 1000) {
@@ -3129,7 +3100,7 @@ class QueryStruct extends QueryNode {
         if (this.unnestWithNumbers() && this.parent !== undefined) {
             const x = this.parent.getSQLIdentifier() +
                 '.' +
-                (0, malloy_types_1.getIdentifier)(this.fieldDef) +
+                (0, malloy_types_1.getIdentifier)(this.structDef) +
                 `[${this.getIdentifier()}.__row_id]`;
             return x;
         }
@@ -3140,12 +3111,12 @@ class QueryStruct extends QueryNode {
     // return the name of the field in SQL
     getIdentifier() {
         // if it is the root table, use provided alias if we have one.
-        if (this.fieldDef.structRelationship.type === 'basetable') {
+        if ((0, malloy_types_1.isBaseTable)(this.structDef)) {
             return 'base';
         }
         // if this is an inline object, include the parents alias.
-        if (this.fieldDef.structRelationship.type === 'inline' && this.parent) {
-            return this.parent.getSQLIdentifier() + '.' + super.getIdentifier();
+        if (this.structDef.type === 'record' && this.parent) {
+            return (this.parent.getSQLIdentifier() + '.' + (0, malloy_types_1.getIdentifier)(this.structDef));
         }
         // we are somewhere in the join tree.  Make sure the alias is unique.
         return this.getAliasIdentifier();
@@ -3153,7 +3124,7 @@ class QueryStruct extends QueryNode {
     // return the name of the field in Malloy
     getFullOutputName() {
         if (this.parent) {
-            return (this.parent.getFullOutputName() + (0, malloy_types_1.getIdentifier)(this.fieldDef) + '.');
+            return (this.parent.getFullOutputName() + (0, malloy_types_1.getIdentifier)(this.structDef) + '.');
         }
         else {
             return '';
@@ -3168,12 +3139,11 @@ class QueryStruct extends QueryNode {
         throw new Error(`Join ${joinName} not found in result set`);
     }
     unnestWithNumbers() {
-        return (this.dialect.unnestWithNumbers &&
-            this.fieldDef.structRelationship.type === 'nested');
+        return this.dialect.unnestWithNumbers && this.structDef.type === 'array';
     }
     getJoinableParent() {
         // if it is inline it should always have a parent
-        if (this.fieldDef.structRelationship.type === 'inline') {
+        if (this.structDef.type === 'record') {
             if (this.parent) {
                 return this.parent.getJoinableParent();
             }
@@ -3205,25 +3175,25 @@ class QueryStruct extends QueryNode {
      * they exist) to the structure.
      */
     resolveQueryFields() {
-        if (this.fieldDef.structSource.type === 'query') {
-            const structDef = this.model
-                .loadQuery(this.fieldDef.structSource.query, undefined, this.prepareResultOptions)
+        if (this.structDef.type === 'query_source') {
+            const resultStruct = this.model
+                .loadQuery(this.structDef.query, undefined, this.prepareResultOptions)
                 .structs.pop();
             // should never happen.
-            if (!structDef) {
+            if (!resultStruct) {
                 throw new Error("Internal Error, query didn't produce a struct");
             }
-            const fieldDef = { ...this.fieldDef };
-            for (const f of structDef.fields) {
-                let as;
-                if (!this.nameMap.has((as = (0, malloy_types_1.getIdentifier)(f)))) {
-                    fieldDef.fields.push(f);
+            const structDef = { ...this.structDef };
+            for (const f of resultStruct.fields) {
+                const as = (0, malloy_types_1.getIdentifier)(f);
+                if (!this.nameMap.has(as)) {
+                    structDef.fields.push(f);
                     this.nameMap.set(as, this.makeQueryField(f));
                 }
             }
-            this.fieldDef = fieldDef;
-            if (!this.fieldDef.primaryKey && structDef.primaryKey) {
-                this.fieldDef.primaryKey = structDef.primaryKey;
+            this.structDef = structDef;
+            if (!this.structDef.primaryKey && resultStruct.primaryKey) {
+                this.structDef.primaryKey = resultStruct.primaryKey;
             }
         }
         for (const [, v] of this.nameMap) {
@@ -3285,53 +3255,37 @@ class QueryStruct extends QueryNode {
     }
     structSourceSQL(stageWriter) {
         var _a, _b;
-        switch (this.fieldDef.structSource.type) {
-            case 'table': {
-                const tablePath = this.fieldDef.structSource.tablePath;
-                return this.dialect.quoteTablePath(tablePath);
-            }
-            case 'sql':
-                if (this.fieldDef.structSource.method === 'nested' ||
-                    this.fieldDef.structSource.method === 'lastStage') {
-                    return this.fieldDef.name;
-                }
-                else if (this.fieldDef.structSource.method === 'subquery') {
-                    return `(${this.fieldDef.structSource.sqlBlock.selectStr})`;
-                }
-                throw new Error("Internal Error: Unknown structSource type 'sql' method");
-            case 'nested':
-                // 'name' is always the source field even if has been renamed through
-                // 'as'
-                return 'UNNEST(this.fieldDef.name)';
-            case 'inline':
-                return '';
-            case 'query': {
+        switch (this.structDef.type) {
+            case 'table':
+                return this.dialect.quoteTablePath(this.structDef.tablePath);
+            case 'finalize':
+                return this.structDef.name;
+            case 'sql_select':
+                return `(${this.structDef.selectStr})`;
+            case 'nest_source':
+                return this.structDef.pipeSQL;
+            case 'query_source': {
                 // cache derived table.
                 if (((_a = this.prepareResultOptions) === null || _a === void 0 ? void 0 : _a.replaceMaterializedReferences) &&
-                    (0, utils_2.shouldMaterialize)(this.fieldDef.structSource.query.annotation)) {
-                    return stageWriter.addMaterializedQuery((0, malloy_types_1.getIdentifier)(this.fieldDef), this.fieldDef.structSource.query, (_b = this.prepareResultOptions) === null || _b === void 0 ? void 0 : _b.materializedTablePrefix);
+                    (0, utils_2.shouldMaterialize)(this.structDef.query.annotation)) {
+                    return stageWriter.addMaterializedQuery((0, malloy_types_1.getIdentifier)(this.structDef), this.structDef.query, (_b = this.prepareResultOptions) === null || _b === void 0 ? void 0 : _b.materializedTablePrefix);
                 }
                 else {
                     // returns the stage name.
-                    return this.model.loadQuery(this.fieldDef.structSource.query, stageWriter, this.prepareResultOptions, false, true // this is an intermediate stage.
+                    return this.model.loadQuery(this.structDef.query, stageWriter, this.prepareResultOptions, false, true // this is an intermediate stage.
                     ).lastStageName;
                 }
             }
             default:
-                throw new Error(`unknown structSource ${this.fieldDef}`);
+                throw new Error(`Cannot create SQL StageWriter from '${(0, malloy_types_1.getIdentifier)(this.structDef)}' type '${this.structDef.type}`);
         }
     }
     root() {
-        if (this.parent === undefined) {
-            return this;
-        }
-        else {
-            return this.parent.root();
-        }
+        return this.parent ? this.parent.root() : this;
     }
     primaryKey() {
-        if (this.fieldDef.primaryKey) {
-            return this.getDimensionByName([this.fieldDef.primaryKey]);
+        if ((0, malloy_types_1.isSourceDef)(this.structDef) && this.structDef.primaryKey) {
+            return this.getDimensionByName([this.structDef.primaryKey]);
         }
         else {
             return undefined;
@@ -3356,25 +3310,26 @@ class QueryStruct extends QueryNode {
     getQueryFieldByName(name) {
         const field = this.getFieldByName(name);
         if (field instanceof QueryStruct) {
-            throw new Error(`Cannot reference ${name} as a scalar'`);
+            throw new Error(`Cannot reference ${name.join('.')} as a scalar'`);
         }
         return field;
     }
-    getQueryFieldReference(name, refAnnoatation) {
-        const field = this.getQueryFieldByName(name);
-        if (refAnnoatation) {
-            // Made the field object from the source, but the annotations were computed by the compiler
-            // and have none of the source and reference annotations included, use those.
+    getQueryFieldReference(name, annotation) {
+        const field = this.getFieldByName(name);
+        if (annotation) {
             if (field.parent === undefined) {
-                throw new Error('Expected field to have a parent');
+                throw new Error('Unexpected reference to orphaned query field');
+            }
+            // Made a field object from the source, but the annotations were computed by the compiler
+            // when it generated the reference, and has both the source and reference annotations included.
+            if (field instanceof QueryStruct) {
+                const newDef = { ...field.structDef, annotation };
+                return new QueryStruct(newDef, undefined, field.parent, {}, field.referenceId);
             }
-            if (!(field instanceof QueryField)) {
-                throw new Error('Expected fieldref to reference a QueryField');
+            else {
+                const newDef = { ...field.fieldDef, annotation };
+                return field.parent.makeQueryField(newDef, field.referenceId);
             }
-            return field.parent.makeQueryField({
-                ...field.fieldDef,
-                annotation: refAnnoatation,
-            }, field.referenceId);
         }
         return field;
     }
@@ -3383,9 +3338,7 @@ class QueryStruct extends QueryNode {
         if (query instanceof QueryAtomicField) {
             return query;
         }
-        else {
-            throw new Error(`${name} is not of type a scalar'`);
-        }
+        throw new Error(`${name} is not an atomic field? Inconceivable!`);
     }
     /** returns a query object for the given name */
     getDimensionByName(name) {
@@ -3393,9 +3346,7 @@ class QueryStruct extends QueryNode {
         if (query instanceof QueryAtomicField && isScalarField(query)) {
             return query;
         }
-        else {
-            throw new Error(`${name} is not of type a scalar'`);
-        }
+        throw new Error(`${name} is not an atomic scalar field? Inconceivable!`);
     }
     /** returns a query object for the given name */
     getStructByName(name) {
@@ -3408,14 +3359,14 @@ class QueryStruct extends QueryNode {
         }
     }
     getDistinctKey() {
-        if (this.fieldDef.structRelationship.type !== 'inline') {
+        if (this.structDef.type !== 'record') {
             return this.getDimensionByName(['__distinct_key']);
         }
         else if (this.parent) {
             return this.parent.getDistinctKey();
         }
         else {
-            throw new Error('Internal Error.  inline struct can not be top level');
+            throw new Error('Asking a record for a primary key? Inconceivable!');
         }
     }
     applyStructFiltersToTurtleDef(turtleDef) {
@@ -3423,7 +3374,7 @@ class QueryStruct extends QueryNode {
         const annotation = turtleDef.annotation;
         const addedFilters = turtleDef.filterList || [];
         pipeline = structuredClone(pipeline);
-        pipeline[0].filterList = addedFilters.concat(pipeline[0].filterList || [], this.fieldDef.filterList || []);
+        pipeline[0].filterList = addedFilters.concat(pipeline[0].filterList || [], (0, malloy_types_1.isSourceDef)(this.structDef) ? this.structDef.filterList || [] : []);
         const flatTurtleDef = {
             type: 'turtle',
             name: turtleDef.name,
@@ -3452,8 +3403,8 @@ class QueryModel {
         this.modelDef = modelDef;
         for (const s of Object.values(this.modelDef.contents)) {
             let qs;
-            if (s.type === 'struct') {
-                qs = new QueryStruct(s, undefined, { model: this });
+            if ((0, malloy_types_1.modelObjIsSource)(s)) {
+                qs = new QueryStruct(s, undefined, { model: this }, {});
                 this.structs.set((0, malloy_types_1.getIdentifier)(s), qs);
                 qs.resolveQueryFields();
             }
@@ -3466,31 +3417,23 @@ class QueryModel {
         }
     }
     getStructByName(name) {
-        let s;
-        if ((s = this.structs.get(name))) {
+        const s = this.structs.get(name);
+        if (s) {
             return s;
         }
-        else {
-            throw new Error(`Struct ${name} not found in model.`);
-        }
+        throw new Error(`Struct ${name} not found in model.`);
     }
     getStructFromRef(structRef, sourceArguments, prepareResultOptions) {
         var _a;
-        let structDef;
+        prepareResultOptions !== null && prepareResultOptions !== void 0 ? prepareResultOptions : (prepareResultOptions = {});
         if (typeof structRef === 'string') {
             const ret = this.getStructByName(structRef);
             if (sourceArguments !== undefined) {
-                return new QueryStruct(ret.fieldDef, sourceArguments, (_a = ret.parent) !== null && _a !== void 0 ? _a : { model: this }, prepareResultOptions);
+                return new QueryStruct(ret.structDef, sourceArguments, (_a = ret.parent) !== null && _a !== void 0 ? _a : { model: this }, prepareResultOptions);
             }
             return ret;
         }
-        else if (structRef.type === 'struct') {
-            structDef = structRef;
-        }
-        else {
-            throw new Error('Broken for now');
-        }
-        return new QueryStruct(structDef, sourceArguments, { model: this }, prepareResultOptions);
+        return new QueryStruct(structRef, sourceArguments, { model: this }, prepareResultOptions);
     }
     loadQuery(query, stageWriter, prepareResultOptions, emitFinalStage = false, isJoinedSubquery = false) {
         const malloy = '';
@@ -3510,7 +3453,7 @@ class QueryModel {
             // for (const f of ret.outputStruct.fields) {
             //   fieldNames.push(getIdentifier(f));
             // }
-            const fieldNames = (0, malloy_types_1.getPhysicalFields)(ret.outputStruct).map(fieldDef => q.parent.dialect.sqlMaybeQuoteIdentifier(fieldDef.name));
+            const fieldNames = (0, malloy_types_1.getAtomicFields)(ret.outputStruct).map(fieldDef => q.parent.dialect.sqlMaybeQuoteIdentifier(fieldDef.name));
             ret.lastStageName = stageWriter.addStage(q.parent.dialect.sqlFinalStage(ret.lastStageName, fieldNames));
         }
         return {
@@ -3528,11 +3471,7 @@ class QueryModel {
         const ret = m.loadQuery(query, undefined, prepareResultOptions, finalize, false);
         const sourceExplore = typeof query.structRef === 'string'
             ? query.structRef
-            : // LTNOTE: the parser needs to capture the query before the |.  This will work
-                //  in most cases but isn't actually complete.
-                query.structRef.type === 'struct'
-                    ? query.structRef.as || query.structRef.name
-                    : '(need to figure this out)';
+            : query.structRef.as || query.structRef.name;
         // LTNote:  I don't understand why this might be here.  It should have happened in loadQuery...
         if (finalize && this.dialect.hasFinalStage) {
             ret.lastStageName = ret.stageWriter.addStage(