@malloydata/malloy 0.0.119-dev240123183113 → 0.0.119-dev240124170348

package/dist/model/malloy_query.js

@@ -27,6 +27,9 @@ const dialect_1 = require("../dialect");
 const standardsql_1 = require("../dialect/standardsql/standardsql");
 const malloy_types_1 = require("./malloy_types");
 const utils_1 = require("./utils");
+function pathToCol(path) {
+    return path.map(el => encodeURIComponent(el)).join('/');
+}
 // quote a string for SQL use.  Perhaps should be in dialect.
 function generateSQLStringLiteral(sourceString) {
     return `'${sourceString}'`;
@@ -330,17 +333,30 @@ class QueryField extends QueryNode {
     generateSpread(_resultSet, _context, _frag, _state) {
         throw new Error('Unexpanded spread encountered during SQL generation');
     }
-    generateParameterFragment(resultSet, context, expr, state) {
+    generateParameterFragment(resultSet, context, expr, _state) {
+        /*
+          mtoy todo parameters and paths figure this out
+    
         // find the structDef and return the path to the field...
         const param = context.parameters()[expr.path];
-        if ((0, malloy_types_1.isValueParameter)(param)) {
-            if (param.value) {
-                return this.generateExpressionFromExpr(resultSet, context, param.value, state);
-            }
-        }
-        else if (param.condition) {
-            return this.generateExpressionFromExpr(resultSet, context, param.condition, state);
-        }
+        if (isValueParameter(param)) {
+          if (param.value) {
+            return this.generateExpressionFromExpr(
+              resultSet,
+              context,
+              param.value,
+              state
+            );
+          }
+        } else if (param.condition) {
+          return this.generateExpressionFromExpr(
+            resultSet,
+            context,
+            param.condition,
+            state
+          );
+        }
+        */
         throw new Error(`Can't generate SQL, no value for ${expr.path}`);
     }
     generateFilterFragment(resultSet, context, expr, state) {
@@ -479,9 +495,6 @@ class QueryField extends QueryNode {
     generateDialect(resultSet, context, expr, state) {
         return this.generateExpressionFromExpr(resultSet, context, context.dialect.dialectExpr(resultSet.getQueryInfo(), expr), state);
     }
-    generateFieldReference(resultSet, context, expr, state) {
-        return this.generateFieldFragment(resultSet, context, { type: 'field', path: expr.path }, state);
-    }
     generateSqlString(resultSet, context, expr, state) {
         return expr.e
             .map(part => typeof part === 'string'
@@ -667,9 +680,6 @@ class QueryField extends QueryNode {
             else if (expr.type === 'source-reference') {
                 s += this.generateSourceReference(resultSet, context, expr);
             }
-            else if (expr.type === 'field-reference') {
-                s += this.generateFieldReference(resultSet, context, expr, state);
-            }
             else {
                 throw new Error(`Internal Error: Unknown expression fragment ${JSON.stringify(expr, undefined, 2)}`);
             }
@@ -699,7 +709,7 @@ function isAggregateField(f) {
     return f instanceof QueryAtomicField && f.isAggregate();
 }
 function isScalarField(f) {
-    return f instanceof QueryAtomicField && !f.isCalculated();
+    return f instanceof QueryAtomicField && !f.isCalculated() && !f.isAggregate();
 }
 class QueryAtomicField extends QueryField {
     includeInWildcard() {
@@ -750,10 +760,10 @@ class QueryFieldStruct extends QueryAtomicField {
                 onExpression: [
                     {
                         type: 'field',
-                        path: this.primaryKey,
+                        path: [this.primaryKey],
                     },
                     '=',
-                    { type: 'field', path: foreignKeyName },
+                    { type: 'field', path: [foreignKeyName] },
                 ],
             },
         };
@@ -1411,136 +1421,14 @@ class QueryQuery extends QueryField {
         return (this.parent.fieldDef.structSource.type === 'sql' &&
             this.parent.fieldDef.structSource.method === 'nested');
     }
-    getFieldList() {
-        switch (this.firstSegment.type) {
-            // case "index":
-            //   return this.firstSegment.fields || [];
-            case 'reduce':
-                return this.firstSegment.fields;
-            // probably need some way of checking type class of field here...
-            //  project should only contain scalars
-            case 'project':
-                return this.firstSegment.fields;
-            default:
-                throw new Error(`Query contains no fields ${JSON.stringify(this.fieldDef)}`);
-        }
-    }
     // get a field ref and expand it.
     expandField(f) {
-        let as;
-        let field;
-        // if it is a string
-        if (typeof f === 'string') {
-            field = this.parent.getQueryFieldByName(f);
-        }
-        else if ('type' in f) {
-            field = this.parent.makeQueryField(f);
-        }
-        // or FilteredAliasedName or a hacked timestamp field.
-        else if ('name' in f && 'as' in f) {
-            field = this.parent.getQueryFieldByName(f.name);
-            // QueryFieldStructs return new names...
-            as = field.fieldDef.as || f.as;
-            if (field instanceof QueryFieldStruct) {
-                throw new Error('Syntax currently disallowed. Semantics up for discussion');
-            }
-            // Types of aliased fields.
-            // turtles
-            // Timestamps and Dates (are just fine to leave as is).
-            // measures
-            // let e: Expr;
-            if (field instanceof QueryQuery) {
-                const newFieldDef = structuredClone(field.fieldDef);
-                newFieldDef.as = f.name;
-                newFieldDef.filterList = f.filterList;
-                field = QueryQuery.makeQuery(newFieldDef, this.parent, undefined, this.isJoinedSubquery);
-            }
-            else if (!(field instanceof QueryFieldTimestamp ||
-                field instanceof QueryFieldDate)) {
-                throw new Error(`No longer generate code this way. \n ${JSON.stringify(f, undefined, 2)}`);
-                // // its a measure
-                // e = [{ type: "field", path: field.getFullOutputName() }];
-                // if ("filterList" in f && f.filterList) {
-                //   e = [{ type: "filterExpression", filterList: f.filterList, e: e }];
-                // }
-                // const newFieldDef = {
-                //   type: field.fieldDef.type,
-                //   name: f.as,
-                //   e,
-                // };
-                // field = this.parent.makeQueryField(newFieldDef as FieldDef);
-            }
-            // or inline field FieldTypeDef
-        }
-        else {
-            throw new Error(`Unrecognized field definition ${JSON.stringify(f, undefined, 2)}`);
-        }
-        if (!as) {
-            as = field.getIdentifier();
-        }
+        const field = f.type === 'fieldref'
+            ? this.parent.getQueryFieldReference(f.path, f.annotation)
+            : this.parent.makeQueryField(f);
+        const as = field.getIdentifier();
         return { as, field };
     }
-    expandDependantField(resultStruct, fieldRef) {
-        this.expandField(fieldRef);
-    }
-    // find all the fieldNames in the struct (and children)
-    //  that match the filter
-    expandWildCardStruct(struct, expandChildren, filter = undefined) {
-        let fieldNames = [];
-        const structs = [];
-        for (const [_name, f] of struct.nameMap) {
-            if (f instanceof QueryAtomicField &&
-                isScalarField(f) &&
-                f.includeInWildcard() &&
-                !this.parent.dialect.ignoreInProject(f.fieldDef.name) &&
-                (!filter || filter(f))) {
-                // fieldNames.push(`${struct.getFullOutputName()}${name}`);
-                fieldNames.push(f.getFullOutputName());
-            }
-            else if (f instanceof QueryStruct && expandChildren) {
-                structs.push(f);
-            }
-        }
-        for (const s of structs) {
-            fieldNames = fieldNames.concat(this.expandWildCardStruct(s, expandChildren, filter));
-        }
-        return fieldNames;
-    }
-    // Do any '*' expansion.
-    expandWildCards(fields, filter = undefined) {
-        let ret = [];
-        for (const f of fields) {
-            if (typeof f !== 'string') {
-                ret.push(f);
-            }
-            else {
-                const fieldName = f;
-                const path = fieldName.split('.');
-                if (!path[path.length - 1].startsWith('*')) {
-                    ret.push(f);
-                }
-                else {
-                    const expandChildren = path.pop() === '**';
-                    let struct = this.parent;
-                    let pathElementName;
-                    while (path.length > 0 && (pathElementName = path.shift())) {
-                        const structNode = struct.getChildByName(pathElementName);
-                        if (structNode === undefined) {
-                            throw new Error(`Nested source not found '${pathElementName}'`);
-                        }
-                        if (structNode instanceof QueryStruct) {
-                            struct = structNode;
-                        }
-                        else {
-                            throw new Error(`'${pathElementName}' is not a source object`);
-                        }
-                    }
-                    ret = ret.concat(this.expandWildCardStruct(struct, expandChildren, filter));
-                }
-            }
-        }
-        return ret;
-    }
     addDependantPath(resultStruct, context, path, uniqueKeyPossibleUse, joinStack) {
         const node = context.getFieldByName(path);
         let struct;
@@ -1668,9 +1556,17 @@ class QueryQuery extends QueryField {
             this.addDependantExpr(resultStruct, field.parent, field.fieldDef.e, []);
         }
     }
+    getSegmentFields(resultStruct) {
+        const fs = resultStruct.firstSegment;
+        return fs.type === 'index'
+            ? fs.indexFields
+            : (0, malloy_types_1.isQuerySegment)(fs)
+                ? fs.queryFields
+                : [];
+    }
     expandFields(resultStruct) {
         let resultIndex = 1;
-        for (const f of this.expandWildCards(resultStruct.firstSegment.fields)) {
+        for (const f of this.getSegmentFields(resultStruct)) {
             const { as, field } = this.expandField(f);
             if (field instanceof QueryTurtle || field instanceof QueryQuery) {
                 if (this.firstSegment.type === 'project') {
@@ -1834,7 +1730,7 @@ class QueryQuery extends QueryField {
                 const resultType = fi.getRepeatedResultType() === 'nested' ? 'nested' : 'inline';
                 structDef.name = name;
                 structDef.structRelationship = {
-                    field: name,
+                    fieldName: name,
                     type: resultType,
                     isArray: false,
                 };
@@ -1998,7 +1894,7 @@ class QueryQuery extends QueryField {
             if (qs.parent === undefined || ji.parent === undefined) {
                 throw new Error('Internal Error, nested structure with no parent.');
             }
-            const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), structRelationship.field, 'struct', qs.parent.fieldDef.structRelationship.type === 'nested', this.parent.fieldDef.structRelationship.type === 'nested' &&
+            const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), structRelationship.fieldName, 'struct', qs.parent.fieldDef.structRelationship.type === 'nested', this.parent.fieldDef.structRelationship.type === 'nested' &&
                 this.parent.fieldDef.structRelationship.isArray);
             // we need to generate primary key.  If parent has a primary key combine
             // console.log(ji.alias, fieldExpression, this.inNestedPipeline());
@@ -2622,20 +2518,21 @@ class QueryQueryProject extends QueryQuery {
 class QueryQueryIndexStage extends QueryQuery {
     constructor(fieldDef, parent, stageWriter, isJoinedSubquery) {
         super(fieldDef, parent, stageWriter, isJoinedSubquery);
+        this.indexPaths = {};
         this.fieldDef = fieldDef;
     }
-    // get a field ref and expand it.
     expandField(f) {
-        const field = this.parent.getFieldByName(f);
-        return { as: f, field };
+        const as = f.path.join('.');
+        const field = this.parent.getQueryFieldByName(f.path);
+        return { as, field };
     }
     expandFields(resultStruct) {
         let resultIndex = 1;
         const groupIndex = resultStruct.groupSet;
         this.maxGroupSet = groupIndex;
-        const fieldNames = this.firstSegment.fields || [];
-        for (const f of fieldNames) {
+        for (const f of this.firstSegment.indexFields) {
             const { as, field } = this.expandField(f);
+            this.indexPaths[as] = f.path;
             resultStruct.addField(as, field, {
                 resultIndex,
                 type: 'result',
@@ -2647,7 +2544,7 @@ class QueryQueryIndexStage extends QueryQuery {
         }
         const measure = this.firstSegment.weightMeasure;
         if (measure !== undefined) {
-            const f = this.parent.getFieldByName(measure);
+            const f = this.parent.getFieldByName([measure]);
             resultStruct.addField(measure, f, {
                 resultIndex,
                 type: 'result',
@@ -2660,6 +2557,7 @@ class QueryQueryIndexStage extends QueryQuery {
         let measureSQL = 'COUNT(*)';
         const dialect = this.parent.dialect;
         const fieldNameColumn = dialect.sqlMaybeQuoteIdentifier('fieldName');
+        const fieldPathColumn = dialect.sqlMaybeQuoteIdentifier('fieldPath');
         const fieldValueColumn = dialect.sqlMaybeQuoteIdentifier('fieldValue');
         const fieldTypeColumn = dialect.sqlMaybeQuoteIdentifier('fieldType');
         const fieldRangeColumn = dialect.sqlMaybeQuoteIdentifier('fieldRange');
@@ -2674,7 +2572,8 @@ class QueryQueryIndexStage extends QueryQuery {
             const fi = field;
             if (fi.fieldUsage.type === 'result' && isScalarField(fi.f)) {
                 const expression = fi.f.generateExpression(this.rootResult);
-                fields.push({ name, type: fi.f.fieldDef.type, expression });
+                const path = this.indexPaths[name] || [];
+                fields.push({ name, path, type: fi.f.fieldDef.type, expression });
             }
         }
         let s = 'SELECT\n  group_set,\n';
@@ -2682,13 +2581,19 @@ class QueryQueryIndexStage extends QueryQuery {
         for (let i = 0; i < fields.length; i++) {
             s += `    WHEN ${i} THEN '${fields[i].name}'\n`;
         }
-        s += `  END as ${fieldNameColumn},`;
+        s += `  END as ${fieldNameColumn},\n`;
+        s += '  CASE group_set\n';
+        for (let i = 0; i < fields.length; i++) {
+            const path = pathToCol(fields[i].path);
+            s += `    WHEN ${i} THEN '${path}'\n`;
+        }
+        s += `  END as ${fieldPathColumn},\n`;
         s += '  CASE group_set\n';
         for (let i = 0; i < fields.length; i++) {
             s += `    WHEN ${i} THEN '${fields[i].type}'\n`;
         }
         s += `  END as ${fieldTypeColumn},`;
-        s += '  CASE group_set WHEN 99999 THEN NULL\n';
+        s += `  CASE group_set WHEN 99999 THEN ${dialect.castToString('NULL')}\n`;
         for (let i = 0; i < fields.length; i++) {
             if (fields[i].type === 'string') {
                 s += `    WHEN ${i} THEN ${fields[i].expression}\n`;
@@ -2717,7 +2622,7 @@ class QueryQueryIndexStage extends QueryQuery {
         s += this.generateSQLJoins(stageWriter);
         s += dialect.sqlGroupSetTable(fields.length) + '\n';
         s += this.generateSQLFilters(this.rootResult, 'where').sql('where');
-        s += 'GROUP BY 1,2,3,4\n';
+        s += 'GROUP BY 1,2,3,4,5\n';
         // limit
         if (!(0, malloy_types_1.isRawSegment)(this.firstSegment) && this.firstSegment.limit) {
             s += `LIMIT ${this.firstSegment.limit}\n`;
@@ -2726,6 +2631,7 @@ class QueryQueryIndexStage extends QueryQuery {
         const resultStage = stageWriter.addStage(s);
         this.resultStage = stageWriter.addStage(`SELECT
   ${fieldNameColumn},
+  ${fieldPathColumn},
   ${fieldTypeColumn},
   COALESCE(${fieldValueColumn}, ${fieldRangeColumn}) as ${fieldValueColumn},
   weight
@@ -2755,90 +2661,57 @@ class QueryQueryRaw extends QueryQuery {
 class QueryQueryIndex extends QueryQuery {
     constructor(fieldDef, parent, stageWriter, isJoinedSubquery) {
         super(fieldDef, parent, stageWriter, isJoinedSubquery);
-        this.rootFields = [];
-        this.fanPrefixMap = {};
+        this.stages = [];
         this.fieldDef = fieldDef;
-        this.findFanPrefexes(parent);
-    }
-    // we want to generate a different query for each
-    //  nested structure so we don't do a crazy cross product.
-    findFanPrefexes(qs) {
-        for (const [_name, f] of qs.nameMap) {
-            if (f instanceof QueryStruct &&
-                (f.fieldDef.structRelationship.type === 'many' ||
-                    f.fieldDef.structRelationship.type === 'nested') &&
-                f.fieldDef.fields.length > 1 && // leave arrays in parent.
-                this.parent.dialect.dontUnionIndex === false) {
-                const key = f.getFullOutputName();
-                this.fanPrefixMap[key] = [];
-                this.findFanPrefexes(f);
-            }
-        }
-    }
-    expandIndexWildCards() {
-        // if no fields were specified, look in the parent struct for strings.
-        let fieldNames = this.firstSegment.fields || [];
-        if (fieldNames.length === 0) {
-            fieldNames.push('**');
-        }
-        fieldNames = this.expandWildCards(fieldNames, qf => ['string', 'number', 'timestamp', 'date'].indexOf(qf.fieldDef.type) !==
-            -1);
-        return fieldNames;
+        this.fieldsToStages();
     }
-    // return the number of stages it is going to take to generate this index.
-    getStageFields() {
-        const s = [];
-        if (this.rootFields.length > 0) {
-            s.push(this.rootFields);
-        }
-        for (const fieldList of Object.values(this.fanPrefixMap)) {
-            if (fieldList.length > 0) {
-                s.push(fieldList);
-            }
+    fieldsToStages() {
+        const indexSeg = this.firstSegment;
+        if (this.parent.dialect.dontUnionIndex) {
+            this.stages = [indexSeg.indexFields];
+            return;
         }
-        return s;
-    }
-    // Map fields into stages based on their level of repeated nesting
-    //
-    mapFieldsIntoStages(fieldNames) {
-        // find all the fanned prefixes, longest ones first.
-        const fannedPrefixes = Object.keys(this.fanPrefixMap).sort((k1, k2) => {
-            if (k1.length < k2.length) {
-                return 1;
-            }
-            if (k1.length > k2.length) {
-                return -1;
-            }
-            return 0;
-        });
-        // Find the deepest fanned prefix
-        for (const fn of fieldNames) {
-            let found = false;
-            for (const prefix of fannedPrefixes) {
-                if (fn.startsWith(prefix)) {
-                    this.fanPrefixMap[prefix].push(fn);
-                    found = true;
+        // Collect the field references by unique path, the final
+        // index will be a union indexes from each unique path
+        const stageMap = {};
+        for (const fref of indexSeg.indexFields) {
+            if (fref.path.length > 1) {
+                const stageRoot = pathToCol(fref.path.slice(0, fref.path.length - 1));
+                const stage = stageMap[stageRoot];
+                if (stage === undefined) {
+                    const f = this.parent.nameMap.get(fref.path[0]);
+                    if (f instanceof QueryStruct &&
+                        (f.fieldDef.structRelationship.type === 'many' ||
+                            f.fieldDef.structRelationship.type === 'nested') &&
+                        f.fieldDef.fields.length > 1) {
+                        const toStage = [fref];
+                        stageMap[stageRoot] = toStage;
+                        this.stages.push(toStage);
+                        continue;
+                    }
+                }
+                else {
+                    stage.push(fref);
+                    continue;
                 }
             }
-            if (!found) {
-                this.rootFields.push(fn);
+            if (this.stages[0] === undefined) {
+                this.stages[0] = [];
             }
+            this.stages[0].push(fref);
         }
     }
-    expandFields(_resultStruct) {
-        const fieldNames = this.expandIndexWildCards();
-        this.mapFieldsIntoStages(fieldNames);
-    }
+    expandFields(_resultStruct) { }
     generateSQL(stageWriter) {
-        const stages = this.getStageFields();
+        const indexSeg = this.firstSegment;
         const outputStageNames = [];
-        for (const fields of stages) {
+        for (const fields of this.stages) {
             const q = new QueryQueryIndexStage({
                 ...this.fieldDef,
                 pipeline: [
                     {
-                        ...this.fieldDef.pipeline[0],
-                        fields: fields,
+                        ...indexSeg,
+                        indexFields: fields,
                     },
                 ],
             }, this.parent, stageWriter, this.isJoinedSubquery);
@@ -2854,7 +2727,11 @@ class QueryQueryIndex extends QueryQuery {
         }
         return this.resultStage;
     }
-    /**  All Indexes have the same output schema */
+    /**
+     * All Indexes have the same output schema.
+     *   fieldName is deprecated, dots in fieldName may or may not be join nodes
+     *   fieldPath is a URL encoded slash separated path
+     */
     getResultStructDef() {
         const ret = {
             type: 'struct',
@@ -2862,6 +2739,7 @@ class QueryQueryIndex extends QueryQuery {
             dialect: this.parent.fieldDef.dialect,
             fields: [
                 { type: 'string', name: 'fieldName' },
+                { type: 'string', name: 'fieldPath' },
                 { type: 'string', name: 'fieldValue' },
                 { type: 'string', name: 'fieldType' },
                 { type: 'number', name: 'weight', numberType: 'integer' },
@@ -2963,32 +2841,6 @@ class QueryStruct extends QueryNode {
             return ret;
         }
     }
-    // when structs are referenced in queries, incorporate the
-    //  primary key of struct and add the struct as a join to the result.
-    getAsQueryField() {
-        if (this.fieldDef.primaryKey === undefined) {
-            throw new Error(`Joined explores can only be included in queries if a primary key is defined: '${this.getFullOutputName()}' has no primary key`);
-        }
-        const pkField = this.getPrimaryKeyField(this.fieldDef);
-        const pkType = pkField.fieldDef.type;
-        if (pkType !== 'string' && pkType !== 'number') {
-            throw new Error(`Unknown Primary key data type for ${pkField.fieldDef.name}`);
-        }
-        const aliasName = (0, malloy_types_1.getIdentifier)(this.fieldDef);
-        const pkName = this.fieldDef.primaryKey;
-        const fieldDef = {
-            type: pkType,
-            name: `${aliasName}_id`,
-            e: [
-                {
-                    type: 'field',
-                    // path: pkField.getFullOutputName(),
-                    path: pkField.getIdentifier(),
-                },
-            ],
-        };
-        return new QueryFieldStruct(fieldDef, this, `${aliasName}.${pkName}`);
-    }
     getSQLIdentifier() {
         if (this.unnestWithNumbers() && this.parent !== undefined) {
             const x = this.parent.getSQLIdentifier() +
@@ -3149,27 +3001,6 @@ class QueryStruct extends QueryNode {
                 throw new Error(`unknown field definition ${JSON.stringify(field)}`);
         }
     }
-    /**
-     * return a field if it exists, make one if we are passed a field definition.
-     */
-    getOrMakeField(fieldRef) {
-        if (typeof fieldRef === 'string') {
-            return this.getFieldByName(fieldRef);
-        }
-        else {
-            return this.makeQueryField(fieldRef);
-        }
-    }
-    /** returns a dimension for the given name  or make one. */
-    getOrMakeDimension(fieldRef) {
-        const dim = this.getOrMakeField(fieldRef);
-        if (dim instanceof QueryAtomicField && isScalarField(dim)) {
-            return dim;
-        }
-        else {
-            throw new Error(`${fieldRef} is not of type a scalar'`);
-        }
-    }
     structSourceSQL(stageWriter) {
         switch (this.fieldDef.structSource.type) {
             case 'table': {
@@ -3221,37 +3052,43 @@ class QueryStruct extends QueryNode {
     }
     primaryKey() {
         if (this.fieldDef.primaryKey) {
-            return this.getDimensionByName(this.fieldDef.primaryKey);
+            return this.getDimensionByName([this.fieldDef.primaryKey]);
         }
         else {
             return undefined;
         }
     }
-    /** get the componennts of a field path */
-    static resolvePath(name) {
-        return name.split('.');
-    }
     getChildByName(name) {
         return this.nameMap.get(name);
     }
-    /** convert a name into a field reference */
-    getFieldByName(name) {
-        const path = QueryStruct.resolvePath(name);
-        let ret = this;
-        for (const n of path) {
-            const r = ret.getChildByName(n);
+    /** convert a path into a field reference */
+    getFieldByName(path) {
+        return path.reduce((lookIn, childName) => {
+            const r = lookIn.getChildByName(childName);
             if (r === undefined) {
-                throw new Error(`Path not found ${name}`);
+                throw new Error(path.length === 1
+                    ? `'${childName}' not found`
+                    : `'${childName}' not found in '${path.join('.')}'`);
             }
-            ret = r;
-        }
-        return ret;
+            return r;
+        }, this);
     }
     // structs referenced in queries are converted to fields.
     getQueryFieldByName(name) {
-        let field = this.getFieldByName(name);
+        const field = this.getFieldByName(name);
         if (field instanceof QueryStruct) {
-            field = field.getAsQueryField();
+            throw new Error(`Cannot reference ${name} as a scalar'`);
+        }
+        return field;
+    }
+    getQueryFieldReference(name, refAnnoatation) {
+        const field = this.getQueryFieldByName(name);
+        if (refAnnoatation) {
+            // Made the field object from the source, but the annotations were computed by the compiler
+            // and have noth the source and reference annotations included, use those.
+            const newDef = { ...field.fieldDef };
+            newDef.annotation = refAnnoatation;
+            field.fieldDef = newDef;
         }
         return field;
     }
@@ -3281,12 +3118,12 @@ class QueryStruct extends QueryNode {
             return struct;
         }
         else {
-            throw new Error(`Error: Path to structure not found '${name}'`);
+            throw new Error(`Error: Path to structure not found '${name.join('.')}'`);
         }
     }
     getDistinctKey() {
         if (this.fieldDef.structRelationship.type !== 'inline') {
-            return this.getDimensionByName('__distinct_key');
+            return this.getDimensionByName(['__distinct_key']);
         }
         else if (this.parent) {
             return this.parent.getDistinctKey();
@@ -3341,22 +3178,6 @@ class QueryModel {
             }
         }
     }
-    parseQueryPath(name) {
-        const path = name.split('.');
-        let struct;
-        if ((struct = this.structs.get(path[0]))) {
-            if (path.length > 1) {
-                path.shift();
-            }
-            else {
-                throw new Error(`No query specified in Struct '${path[0]}'`);
-            }
-            return { queryName: path.join('.'), struct };
-        }
-        else {
-            throw new Error(`Cannot find Struct '${path[0]}' Model`);
-        }
-    }
     getStructByName(name) {
         let s;
         if ((s = this.structs.get(name))) {
@@ -3443,17 +3264,27 @@ class QueryModel {
         }
         // make a search index if one isn't modelled.
         const struct = this.getStructByName(explore);
+        let indexStar = [];
+        for (const [fn, fv] of struct.nameMap) {
+            if (!(fv instanceof QueryStruct)) {
+                if (isScalarField(fv) && fv.includeInWildcard()) {
+                    indexStar.push({ type: 'fieldref', path: [fn] });
+                }
+            }
+        }
+        indexStar = indexStar.sort((a, b) => a.path[0].localeCompare(b.path[0]));
         const indexQuery = {
             structRef: explore,
             pipeline: [
                 {
                     type: 'index',
-                    fields: ['*'],
+                    indexFields: indexStar,
                     sample: struct.dialect.defaultSampling,
                 },
             ],
         };
         const fieldNameColumn = struct.dialect.sqlMaybeQuoteIdentifier('fieldName');
+        const fieldPathColumn = struct.dialect.sqlMaybeQuoteIdentifier('fieldPath');
         const fieldValueColumn = struct.dialect.sqlMaybeQuoteIdentifier('fieldValue');
         const fieldTypeColumn = struct.dialect.sqlMaybeQuoteIdentifier('fieldType');
         // if we've compiled the SQL before use it otherwise
@@ -3464,10 +3295,11 @@ class QueryModel {
         }
         let query = `SELECT
               ${fieldNameColumn},
+              ${fieldPathColumn},
               ${fieldValueColumn},
               ${fieldTypeColumn},
               weight,
-              CASE WHEN lower(${fieldValueColumn}) LIKE  lower(${generateSQLStringLiteral(searchValue + '%')}) THEN 1 ELSE 0 END as match_first
+              CASE WHEN lower(${fieldValueColumn}) LIKE lower(${generateSQLStringLiteral(searchValue + '%')}) THEN 1 ELSE 0 END as match_first
             FROM  ${await connection.manifestTemporaryTable(sqlPDT)}
             WHERE lower(${fieldValueColumn}) LIKE lower(${generateSQLStringLiteral('%' + searchValue + '%')}) ${searchField !== undefined
             ? ` AND ${fieldNameColumn} = '` + searchField + "' \n"
@@ -3478,6 +3310,7 @@ class QueryModel {
         if (struct.dialect.hasFinalStage) {
             query = `WITH __stage0 AS(\n${query}\n)\n${struct.dialect.sqlFinalStage('__stage0', [
                 fieldNameColumn,
+                fieldPathColumn,
                 fieldValueColumn,
                 fieldTypeColumn,
                 'weight',