@malloydata/malloy 0.0.119-dev240123183113 → 0.0.119-dev240124170348

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 export { DuckDBDialect, StandardSQLDialect, PostgresDialect, registerDialect, arg, qtz, overload, minScalar, anyExprType, minAggregate, maxScalar, sql, makeParam, sqlFragment, param, spread, Dialect, FUNCTIONS, } from './dialect';
 export type { DialectFieldList, DialectFunctionOverloadDef, QueryInfo, } from './dialect';
-export type { QueryDataRow, Fragment, StructDef, StructRelationship, NamedStructDefs, MalloyQueryData, AtomicFieldType as AtomicFieldTypeInner, DateUnit, ExtractUnit, TimestampUnit, TimeFieldType, QueryData, QueryValue, FieldTypeDef, Expr, DialectFragment, TimeValue, FilterExpression, SQLBlock, FieldAtomicDef, FieldDef, FilteredAliasedName, PipeSegment, QueryFieldDef, TurtleDef, SearchValueMapResult, SearchIndexResult, ModelDef, Query, QueryResult, QueryRunStats, NamedQuery, NamedModelObject, ExpressionType, FunctionDef, FunctionOverloadDef, FunctionParameterDef, ExpressionValueType, TypeDesc, FieldValueType, ExpressionTypeDesc, FunctionParamTypeDesc, DocumentLocation, DocumentRange, DocumentPosition, Sampling, TypecastFragment, Annotation, FieldAtomicTypeDef, SQLBlockStructDef, } from './model';
-export { Segment, isFilteredAliasedName, isSamplingEnable, isSamplingPercent, isSamplingRows, mkExpr, expressionIsCalculation, indent, } from './model';
+export type { QueryDataRow, Fragment, StructDef, StructRelationship, NamedStructDefs, MalloyQueryData, AtomicFieldType as AtomicFieldTypeInner, DateUnit, ExtractUnit, TimestampUnit, TimeFieldType, QueryData, QueryValue, FieldTypeDef, Expr, DialectFragment, TimeValue, FilterExpression, SQLBlock, FieldAtomicDef, FieldDef, PipeSegment, QueryFieldDef, IndexFieldDef, TurtleDef, SearchValueMapResult, SearchIndexResult, ModelDef, Query, QueryResult, QueryRunStats, NamedQuery, NamedModelObject, ExpressionType, FunctionDef, FunctionOverloadDef, FunctionParameterDef, ExpressionValueType, TypeDesc, FieldValueType, ExpressionTypeDesc, FunctionParamTypeDesc, DocumentLocation, DocumentRange, DocumentPosition, Sampling, TypecastFragment, Annotation, FieldAtomicTypeDef, SQLBlockStructDef, } from './model';
+export { Segment, isSamplingEnable, isSamplingPercent, isSamplingRows, mkExpr, expressionIsCalculation, indent, } from './model';
 export { HighlightType, MalloyTranslator, } from './lang';
 export type { LogMessage, TranslateResponse } from './lang';
 export { Malloy, Runtime, AtomicFieldType, ConnectionRuntime, SingleConnectionRuntime, EmptyURLReader, InMemoryURLReader, FixedConnectionMap, MalloyError, JoinRelationship, SourceRelationship, DateTimeframe, TimestampTimeframe, PreparedResult, Result, QueryMaterializer, CSVWriter, JSONWriter, Parse, DataWriter, Explore, } from './malloy';

package/dist/index.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toAsyncGenerator = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.AtomicFieldType = exports.Runtime = exports.Malloy = exports.MalloyTranslator = exports.HighlightType = exports.indent = exports.expressionIsCalculation = exports.mkExpr = exports.isSamplingRows = exports.isSamplingPercent = exports.isSamplingEnable = exports.isFilteredAliasedName = exports.Segment = exports.FUNCTIONS = exports.Dialect = exports.spread = exports.param = exports.sqlFragment = exports.makeParam = exports.sql = exports.maxScalar = exports.minAggregate = exports.anyExprType = exports.minScalar = exports.overload = exports.qtz = exports.arg = exports.registerDialect = exports.PostgresDialect = exports.StandardSQLDialect = exports.DuckDBDialect = void 0;
-exports.Tag = void 0;
+exports.Tag = exports.toAsyncGenerator = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.AtomicFieldType = exports.Runtime = exports.Malloy = exports.MalloyTranslator = exports.HighlightType = exports.indent = exports.expressionIsCalculation = exports.mkExpr = exports.isSamplingRows = exports.isSamplingPercent = exports.isSamplingEnable = exports.Segment = exports.FUNCTIONS = exports.Dialect = exports.spread = exports.param = exports.sqlFragment = exports.makeParam = exports.sql = exports.maxScalar = exports.minAggregate = exports.anyExprType = exports.minScalar = exports.overload = exports.qtz = exports.arg = exports.registerDialect = exports.PostgresDialect = exports.StandardSQLDialect = exports.DuckDBDialect = void 0;
 /*
  * Copyright 2023 Google LLC
  *
@@ -46,7 +45,6 @@ Object.defineProperty(exports, "FUNCTIONS", { enumerable: true, get: function ()
 var model_1 = require("./model");
 // Used in Composer Demo
 Object.defineProperty(exports, "Segment", { enumerable: true, get: function () { return model_1.Segment; } });
-Object.defineProperty(exports, "isFilteredAliasedName", { enumerable: true, get: function () { return model_1.isFilteredAliasedName; } });
 Object.defineProperty(exports, "isSamplingEnable", { enumerable: true, get: function () { return model_1.isSamplingEnable; } });
 Object.defineProperty(exports, "isSamplingPercent", { enumerable: true, get: function () { return model_1.isSamplingPercent; } });
 Object.defineProperty(exports, "isSamplingRows", { enumerable: true, get: function () { return model_1.isSamplingRows; } });

package/dist/lang/ast/error-factory.js

@@ -51,13 +51,13 @@ class ErrorFactory {
         };
     }
     static get reduceSegment() {
-        return { type: 'reduce', fields: [] };
+        return { type: 'reduce', queryFields: [] };
     }
     static get projectSegment() {
-        return { type: 'project', fields: [] };
+        return { type: 'project', queryFields: [] };
     }
     static get indexSegment() {
-        return { type: 'index', fields: [] };
+        return { type: 'index', indexFields: [] };
     }
 }
 exports.ErrorFactory = ErrorFactory;

package/dist/lang/ast/expressions/expr-aggregate-function.d.ts

@@ -3,6 +3,10 @@ import { FieldReference } from '../query-items/field-references';
 import { ExprValue } from '../types/expr-value';
 import { ExpressionDef } from '../types/expression-def';
 import { FieldSpace } from '../types/field-space';
+interface JoinPathElement {
+    name: string;
+    structRelationship: StructRelationship;
+}
 export declare abstract class ExprAggregateFunction extends ExpressionDef {
     readonly func: AggregateFunctionType;
     elementType: string;
@@ -14,8 +18,6 @@ export declare abstract class ExprAggregateFunction extends ExpressionDef {
     returns(_forExpression: ExprValue): FieldValueType;
     getExpression(fs: FieldSpace): ExprValue;
     isSymmetricFunction(): boolean;
-    getJoinUsage(fs: FieldSpace): {
-        name: string;
-        structRelationship: StructRelationship;
-    }[][];
+    getJoinUsage(fs: FieldSpace): JoinPathElement[][];
 }
+export {};

package/dist/lang/ast/expressions/expr-aggregate-function.js

@@ -55,7 +55,7 @@ class ExprAggregateFunction extends expression_def_1.ExpressionDef {
         const inputFS = fs.isQueryFieldSpace() ? fs.inputSpace() : fs;
         let expr = this.expr;
         let exprVal = (_a = this.expr) === null || _a === void 0 ? void 0 : _a.getExpression(inputFS);
-        let structPath = (_b = this.source) === null || _b === void 0 ? void 0 : _b.refString;
+        let structPath = (_b = this.source) === null || _b === void 0 ? void 0 : _b.path;
         let sourceRelationship = [];
         if (this.source) {
             const result = this.source.getField(inputFS);
@@ -76,21 +76,20 @@ class ExprAggregateFunction extends expression_def_1.ExpressionDef {
                                 }
                                 : {
                                     type: 'field',
-                                    path: this.source.refString,
+                                    path: this.source.path,
                                 },
                         ],
                         evalSpace: footType.evalSpace,
                     };
-                    structPath = this.source.sourceString;
+                    structPath = this.source.path;
                     // Here we handle a special case where you write `foo.agg()` and `foo` is a
                     // dimension which uses only one distinct join path; in this case, we set the
                     // locality to be that join path
                     const joinUsage = this.getJoinUsage(inputFS);
-                    const allUsagePaths = joinUsage.map(x => x.map(y => y.name).join('.'));
-                    const allUsagesSame = allUsagePaths.length > 0 &&
-                        allUsagePaths.slice(1).every(x => x === allUsagePaths[0]);
+                    const allUsagesSame = joinUsage.length === 1 ||
+                        joinUsage.slice(1).every(p => joinPathEq(p, joinUsage[0]));
                     if (allUsagesSame) {
-                        structPath = allUsagePaths[0];
+                        structPath = joinUsage[0].map(p => p.name);
                         sourceRelationship = joinUsage[0];
                     }
                 }
@@ -144,7 +143,7 @@ class ExprAggregateFunction extends expression_def_1.ExpressionDef {
                 function: this.func,
                 e: exprVal.value,
             };
-            if (structPath) {
+            if (structPath && structPath.length > 0) {
                 f.structPath = structPath;
             }
             return {
@@ -164,7 +163,7 @@ class ExprAggregateFunction extends expression_def_1.ExpressionDef {
         if (this.source) {
             const lookup = this.source.getField(fs);
             if (lookup.found) {
-                const sfd = [{ type: 'field', path: this.source.refString }];
+                const sfd = [{ type: 'field', path: this.source.path }];
                 result.push(...getJoinUsage(fs, sfd));
             }
         }
@@ -176,18 +175,30 @@ class ExprAggregateFunction extends expression_def_1.ExpressionDef {
     }
 }
 exports.ExprAggregateFunction = ExprAggregateFunction;
+function joinPathEq(a1, a2) {
+    let len = a1.length;
+    if (len !== a2.length) {
+        return false;
+    }
+    while (len > 0) {
+        len -= 1;
+        if (a1[len].name !== a2[len].name) {
+            return false;
+        }
+    }
+    return true;
+}
 function getJoinUsage(fs, expr) {
     const result = [];
     const lookup = (fs, path) => {
-        const parts = path.split('.');
-        const head = parts[0];
-        const rest = parts.slice(1);
+        const head = path[0];
+        const rest = path.slice(1);
         const def = fs.entry(head);
         if (def === undefined) {
             throw new Error(`Invalid field lookup ${head}`);
         }
         if (def instanceof static_space_1.StructSpaceField && rest.length > 0) {
-            const restDef = lookup(def.fieldSpace, rest.join('.'));
+            const restDef = lookup(def.fieldSpace, rest);
             return {
                 ...restDef,
                 relationship: [

package/dist/lang/ast/expressions/expr-count.js

@@ -44,7 +44,7 @@ class ExprCount extends expr_aggregate_function_1.ExprAggregateFunction {
             e: [],
         };
         if (this.source) {
-            ret.structPath = this.source.refString;
+            ret.structPath = this.source.path;
         }
         return {
             dataType: 'number',

package/dist/lang/ast/expressions/expr-func.js

@@ -82,7 +82,7 @@ class ExprFunc extends expression_def_1.ExpressionDef {
         // Find the 'implicit argument' for aggregate functions called like `some_join.some_field.agg(...args)`
         // where the full arg list is `(some_field, ...args)`.
         let implicitExpr = undefined;
-        let structPath = (_d = this.source) === null || _d === void 0 ? void 0 : _d.refString;
+        let structPath = (_d = this.source) === null || _d === void 0 ? void 0 : _d.path;
         if (this.source) {
             const sourceFoot = this.source.getField(fs).found;
             if (sourceFoot) {
@@ -91,10 +91,10 @@ class ExprFunc extends expression_def_1.ExpressionDef {
                     implicitExpr = {
                         dataType: footType.dataType,
                         expressionType: footType.expressionType,
-                        value: [{ type: 'field', path: this.source.refString }],
+                        value: [{ type: 'field', path: this.source.path }],
                         evalSpace: footType.evalSpace,
                     };
-                    structPath = this.source.sourceString;
+                    structPath = this.source.path.slice(0, -1);
                 }
                 else {
                     if (!(sourceFoot instanceof struct_space_field_base_1.StructSpaceFieldBase)) {
@@ -209,7 +209,7 @@ class ExprFunc extends expression_def_1.ExpressionDef {
                             ? part.value
                             : part.name === 'TABLE'
                                 ? { type: 'source-reference' }
-                                : { type: 'field-reference', path: part.name }),
+                                : { type: 'field', path: [part.name] }),
                     },
                 ];
             }

package/dist/lang/ast/expressions/expr-id-reference.js

@@ -48,7 +48,7 @@ class ExprIdReference extends expression_def_1.ExpressionDef {
                     value: [{ type: 'outputField', name: this.refString }],
                 };
             }
-            const value = [{ type: def.found.refType, path: this.refString }];
+            const value = [{ type: def.found.refType, path: this.fieldReference.path }];
             // We think that aggregates are more 'output' like, but maybe we will reconsider that...
             const evalSpace = (0, malloy_types_1.expressionIsAggregate)(td.expressionType)
                 ? 'output'
@@ -73,7 +73,7 @@ class ExprIdReference extends expression_def_1.ExpressionDef {
                         {
                             type: 'apply',
                             value: lval.value,
-                            to: [{ type: 'parameter', path: this.refString }],
+                            to: [{ type: 'parameter', path: this.fieldReference.path }],
                         },
                     ],
                 };

package/dist/lang/ast/field-space/dynamic-space.js

@@ -68,7 +68,7 @@ class DynamicSpace extends static_space_1.StaticSpace {
         this.setEntry(newName, entry);
     }
     addFieldDef(fd) {
-        this.setEntry((0, field_utils_1.nameOf)(fd), this.defToSpaceField(fd));
+        this.setEntry((0, field_utils_1.nameFromDef)(fd), this.defToSpaceField(fd));
     }
     setTimezone(tz) {
         this.newTimezone = tz;

package/dist/lang/ast/field-space/index-field-space.d.ts

@@ -1,9 +1,11 @@
 import { IndexSegment, PipeSegment } from '../../../model/malloy_types';
+import { WildcardFieldReference } from '../query-items/field-references';
 import { MalloyElement } from '../types/malloy-element';
-import { QuerySpace } from './query-spaces';
-export declare class IndexFieldSpace extends QuerySpace {
+import { QueryOperationSpace } from './query-spaces';
+export declare class IndexFieldSpace extends QueryOperationSpace {
     readonly segmentType = "index";
-    fieldList: Set<string>;
     pushFields(...defs: MalloyElement[]): void;
     getPipeSegment(refineIndex?: PipeSegment): IndexSegment;
+    addRefineFromFields(_refineThis: never): void;
+    protected addWild(wild: WildcardFieldReference): void;
 }

package/dist/lang/ast/field-space/index-field-space.js

@@ -23,24 +23,24 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IndexFieldSpace = void 0;
+const malloy_types_1 = require("../../../model/malloy_types");
 const field_references_1 = require("../query-items/field-references");
+const space_field_1 = require("../types/space-field");
 const query_spaces_1 = require("./query-spaces");
-class IndexFieldSpace extends query_spaces_1.QuerySpace {
+const reference_field_1 = require("./reference-field");
+const static_space_1 = require("./static-space");
+class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
     constructor() {
         super(...arguments);
         this.segmentType = 'index';
-        this.fieldList = new Set();
     }
     pushFields(...defs) {
         for (const indexField of defs) {
             if (indexField instanceof field_references_1.FieldReference) {
-                if (indexField.getField(this.inputSpace()).found) {
-                    this.fieldList.add(indexField.refString);
-                }
-                // mtoy TODO else error ???
+                super.pushFields(indexField);
             }
             else if (indexField instanceof field_references_1.WildcardFieldReference) {
-                this.fieldList.add(indexField.refString);
+                this.addWild(indexField);
             }
             else {
                 indexField.log('Internal error, not expected in index query');
@@ -48,18 +48,87 @@ class IndexFieldSpace extends query_spaces_1.QuerySpace {
         }
     }
     getPipeSegment(refineIndex) {
-        if (refineIndex && refineIndex.fields) {
-            for (const exists of refineIndex.fields) {
-                if (typeof exists === 'string') {
-                    this.fieldList.add(exists);
+        if (refineIndex) {
+            this.log('index query operations cannot be refined');
+            return { type: 'index', indexFields: [] };
+        }
+        const indexFields = [];
+        for (const [name, field] of this.entries()) {
+            if (field instanceof space_field_1.SpaceField) {
+                const wildPath = this.expandedWild[name];
+                if (wildPath) {
+                    indexFields.push({ type: 'fieldref', path: wildPath });
+                    continue;
+                }
+                if (field instanceof reference_field_1.ReferenceField) {
+                    // attempt to cause a type check
+                    const fieldRef = field.fieldRef;
+                    const check = fieldRef.getField(this.exprSpace);
+                    if (check.error) {
+                        fieldRef.log(check.error);
+                    }
+                    else {
+                        indexFields.push(fieldRef.refToField);
+                    }
+                }
+            }
+        }
+        return { type: 'index', indexFields };
+    }
+    addRefineFromFields(_refineThis) { }
+    addWild(wild) {
+        var _a;
+        let current = this.exprSpace;
+        const joinPath = [];
+        if (wild.joinPath) {
+            // walk path to determine namespace for *
+            for (const pathPart of wild.joinPath.list) {
+                const part = pathPart.refString;
+                joinPath.push(part);
+                const ent = current.entry(part);
+                if (ent) {
+                    if (ent instanceof static_space_1.StructSpaceField) {
+                        current = ent.fieldSpace;
+                    }
+                    else {
+                        pathPart.log(`Field '${part}' does not contain rows and cannot be expanded with '*'`);
+                        return;
+                    }
+                }
+                else {
+                    pathPart.log(`No such field as '${part}'`);
+                    return;
                 }
             }
         }
-        this.isComplete();
-        return {
-            type: 'index',
-            fields: Array.from(this.fieldList.values()),
-        };
+        const dialect = this.dialectObj();
+        const expandEntries = [];
+        for (const [name, entry] of current.entries()) {
+            if (wild.except.has(name)) {
+                continue;
+            }
+            const indexName = field_references_1.IndexFieldReference.indexOutputName([
+                ...joinPath,
+                name,
+            ]);
+            if (this.entry(indexName)) {
+                const conflict = (_a = this.expandedWild[indexName]) === null || _a === void 0 ? void 0 : _a.join('.');
+                wild.log(`Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
+            }
+            else {
+                const eType = entry.typeDesc();
+                if ((0, malloy_types_1.isAtomicFieldType)(eType.dataType) &&
+                    (0, malloy_types_1.expressionIsScalar)(eType.expressionType) &&
+                    (dialect === undefined || !dialect.ignoreInProject(name))) {
+                    expandEntries.push({ name: indexName, entry });
+                    this.expandedWild[indexName] = joinPath.concat(name);
+                }
+            }
+        }
+        // There were tests which expected these to be sorted, and that seems reasonable
+        for (const x of expandEntries.sort((a, b) => a.name.localeCompare(b.name))) {
+            this.setEntry(x.name, x.entry);
+        }
     }
 }
 exports.IndexFieldSpace = IndexFieldSpace;

package/dist/lang/ast/field-space/query-input-space.d.ts

@@ -9,7 +9,7 @@ import { AtomicFieldDeclaration } from '../query-items/field-declaration';
 import { Join } from '../source-properties/joins';
 import { SourceSpec } from '../space-seed';
 import { QueryFieldSpace } from '../types/field-space';
-import { QuerySpace } from './query-spaces';
+import { QueryOperationSpace } from './query-spaces';
 import { RefinedSpace } from './refined-space';
 export declare class QueryInputSpace extends RefinedSpace implements QueryFieldSpace {
     private queryOutput;
@@ -20,9 +20,9 @@ export declare class QueryInputSpace extends RefinedSpace implements QueryFieldS
      * @param input The source which might be extended
      * @param queryOutput MUST BE A QuerySpace
      */
-    constructor(input: SourceSpec, queryOutput: QuerySpace);
+    constructor(input: SourceSpec, queryOutput: QueryOperationSpace);
     extendSource(extendField: Join | AtomicFieldDeclaration): void;
     isQueryFieldSpace(): boolean;
-    outputSpace(): QuerySpace;
+    outputSpace(): QueryOperationSpace;
     inputSpace(): this;
 }

package/dist/lang/ast/field-space/query-spaces.d.ts

@@ -6,31 +6,34 @@ import { RefinedSpace } from './refined-space';
 import { LookupResult } from '../types/lookup-result';
 import { QueryInputSpace } from './query-input-space';
 /**
- * The output space of a query operation, it is not named "QueryOutputSpace"
- * because this is the namespace of the Query. This is the one which is constructed
- * with the query. The QueryInputSpace is created and paired when a
- * QuerySpace is created.
+ * The output space of a query operation. It is not named "QueryOutputSpace"
+ * because this is the namespace of the Query which is a layer of an output and
+ * an input space. It constructed with the query operation. A QueryInputSpace is
+ * created and paired when a QueryOperationSpace is created.
  */
-export declare abstract class QuerySpace extends RefinedSpace implements QueryFieldSpace {
+export declare abstract class QueryOperationSpace extends RefinedSpace implements QueryFieldSpace {
     readonly queryInputSpace: FieldSpace;
-    readonly nestParent: QuerySpace | undefined;
+    readonly nestParent: QueryOperationSpace | undefined;
     readonly astEl: MalloyElement;
-    private exprSpace;
+    protected exprSpace: QueryInputSpace;
     abstract readonly segmentType: 'reduce' | 'project' | 'index';
     expandedWild: Record<string, string[]>;
-    constructor(queryInputSpace: FieldSpace, refineThis: model.PipeSegment | undefined, nestParent: QuerySpace | undefined, astEl: MalloyElement);
-    private addRefineFromFields;
+    constructor(queryInputSpace: FieldSpace, refineThis: model.PipeSegment | undefined, nestParent: QueryOperationSpace | undefined, astEl: MalloyElement);
+    abstract addRefineFromFields(refineThis: model.PipeSegment): void;
     log(s: string): void;
-    pushFields(...defs: MalloyElement[]): void;
+    inputSpace(): QueryInputSpace;
+    outputSpace(): QueryOperationSpace;
     protected addWild(wild: WildcardFieldReference): void;
+}
+export declare abstract class QuerySpace extends QueryOperationSpace {
+    addRefineFromFields(refineThis: model.PipeSegment): void;
+    pushFields(...defs: MalloyElement[]): void;
     canContain(_typeDesc: model.TypeDesc): boolean;
     protected queryFieldDefs(): model.QueryFieldDef[];
     getQuerySegment(rf: model.QuerySegment | undefined): model.QuerySegment;
     getPipeSegment(refineFrom: model.QuerySegment | undefined): model.PipeSegment;
     lookup(path: FieldName[]): LookupResult;
     isQueryFieldSpace(): boolean;
-    outputSpace(): this;
-    inputSpace(): QueryInputSpace;
 }
 export declare class ReduceFieldSpace extends QuerySpace {
     readonly segmentType = "reduce";

package/dist/lang/ast/field-space/query-spaces.js

@@ -45,9 +45,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ReduceFieldSpace = exports.QuerySpace = void 0;
+exports.ReduceFieldSpace = exports.QuerySpace = exports.QueryOperationSpace = void 0;
 const model = __importStar(require("../../../model/malloy_types"));
 const field_utils_1 = require("../../field-utils");
+const field_space_1 = require("../types/field-space");
 const space_field_1 = require("../types/space-field");
 const field_references_1 = require("../query-items/field-references");
 const refined_space_1 = require("./refined-space");
@@ -55,12 +56,12 @@ const column_space_field_1 = require("./column-space-field");
 const static_space_1 = require("./static-space");
 const query_input_space_1 = require("./query-input-space");
 /**
- * The output space of a query operation, it is not named "QueryOutputSpace"
- * because this is the namespace of the Query. This is the one which is constructed
- * with the query. The QueryInputSpace is created and paired when a
- * QuerySpace is created.
+ * The output space of a query operation. It is not named "QueryOutputSpace"
+ * because this is the namespace of the Query which is a layer of an output and
+ * an input space. It constructed with the query operation. A QueryInputSpace is
+ * created and paired when a QueryOperationSpace is created.
  */
-class QuerySpace extends refined_space_1.RefinedSpace {
+class QueryOperationSpace extends refined_space_1.RefinedSpace {
     constructor(queryInputSpace, refineThis, nestParent, astEl) {
         super(queryInputSpace.emptyStructDef());
         this.queryInputSpace = queryInputSpace;
@@ -71,45 +72,16 @@ class QuerySpace extends refined_space_1.RefinedSpace {
         if (refineThis)
             this.addRefineFromFields(refineThis);
     }
-    addRefineFromFields(refineThis) {
-        var _a, _b;
-        for (const field of refineThis.fields) {
-            if (typeof field === 'string') {
-                const ent = this.exprSpace.entry(field);
-                if (ent) {
-                    this.setEntry(field, ent);
-                }
-            }
-            else if (model.isFilteredAliasedName(field)) {
-                const name = (_a = field.as) !== null && _a !== void 0 ? _a : field.name;
-                const ent = this.exprSpace.entry(name);
-                if (ent) {
-                    this.setEntry(name, ent);
-                }
-            }
-            else {
-                // TODO can you reference fields in a turtle as fields in the output space,
-                // e.g. order_by: my_turtle.foo, or lag(my_turtle.foo)
-                if (field.type !== 'turtle') {
-                    this.setEntry((_b = field.as) !== null && _b !== void 0 ? _b : field.name, new column_space_field_1.ColumnSpaceField(field));
-                }
-            }
-        }
-    }
     log(s) {
         if (this.astEl) {
             this.astEl.log(s);
         }
     }
-    pushFields(...defs) {
-        for (const f of defs) {
-            if (f instanceof field_references_1.WildcardFieldReference) {
-                this.addWild(f);
-            }
-            else {
-                super.pushFields(f);
-            }
-        }
+    inputSpace() {
+        return this.exprSpace;
+    }
+    outputSpace() {
+        return this;
     }
     addWild(wild) {
         var _a;
@@ -137,6 +109,7 @@ class QuerySpace extends refined_space_1.RefinedSpace {
             }
         }
         const dialect = this.dialectObj();
+        const expandEntries = [];
         for (const [name, entry] of current.entries()) {
             if (wild.except.has(name)) {
                 continue;
@@ -150,11 +123,49 @@ class QuerySpace extends refined_space_1.RefinedSpace {
                 if (model.isAtomicFieldType(eType.dataType) &&
                     model.expressionIsScalar(eType.expressionType) &&
                     (dialect === undefined || !dialect.ignoreInProject(name))) {
-                    this.setEntry(name, entry);
+                    expandEntries.push({ name, entry });
                     this.expandedWild[name] = joinPath.concat(name);
                 }
             }
         }
+        // There were tests which expected these to be sorted, and that seems reasonable
+        for (const x of expandEntries.sort((a, b) => a.name.localeCompare(b.name))) {
+            this.setEntry(x.name, x.entry);
+        }
+    }
+}
+exports.QueryOperationSpace = QueryOperationSpace;
+// Project and Reduce or "QuerySegments" are built from a QuerySpace
+class QuerySpace extends QueryOperationSpace {
+    addRefineFromFields(refineThis) {
+        var _a;
+        if (!model.isQuerySegment(refineThis)) {
+            // TODO mtoy raw,partial,index
+            return;
+        }
+        for (const field of refineThis.queryFields) {
+            if (field.type === 'fieldref') {
+                const refTo = this.exprSpace.lookup(field.path.map(f => new field_space_1.FieldName(f)));
+                if (refTo.found) {
+                    this.setEntry(field.path[field.path.length - 1], refTo.found);
+                }
+            }
+            else if (field.type !== 'turtle') {
+                // TODO can you reference fields in a turtle as fields in the output space,
+                // e.g. order_by: my_turtle.foo, or lag(my_turtle.foo)
+                this.setEntry((_a = field.as) !== null && _a !== void 0 ? _a : field.name, new column_space_field_1.ColumnSpaceField(field));
+            }
+        }
+    }
+    pushFields(...defs) {
+        for (const f of defs) {
+            if (f instanceof field_references_1.WildcardFieldReference) {
+                this.addWild(f);
+            }
+            else {
+                super.pushFields(f);
+            }
+        }
     }
     canContain(_typeDesc) {
         return true;
@@ -165,7 +176,7 @@ class QuerySpace extends refined_space_1.RefinedSpace {
             if (field instanceof space_field_1.SpaceField) {
                 const wildPath = this.expandedWild[name];
                 if (wildPath) {
-                    fields.push(wildPath.join('.'));
+                    fields.push({ type: 'fieldref', path: wildPath });
                     continue;
                 }
                 const fieldQueryDef = field.getQueryFieldDef(this.exprSpace);
@@ -202,8 +213,9 @@ class QuerySpace extends refined_space_1.RefinedSpace {
     }
     getPipeSegment(refineFrom) {
         if (this.segmentType === 'index') {
-            // TODO ... should make this go away
-            throw new Error('INDEX FIELD PIPE SEGMENT MIS HANDLED');
+            // come coding error made this "impossible" thing happen
+            this.log('internal error generating index segment from non index query');
+            return { type: 'reduce', queryFields: [] };
         }
         if (refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.extendSource) {
             for (const xField of refineFrom.extendSource) {
@@ -212,9 +224,9 @@ class QuerySpace extends refined_space_1.RefinedSpace {
         }
         const segment = {
             type: this.segmentType,
-            fields: this.queryFieldDefs(),
+            queryFields: this.queryFieldDefs(),
         };
-        segment.fields = (0, field_utils_1.mergeFields)(refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.fields, segment.fields);
+        segment.queryFields = (0, field_utils_1.mergeFields)(refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.queryFields, segment.queryFields);
         if (refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.extendSource) {
             segment.extendSource = refineFrom.extendSource;
         }
@@ -222,7 +234,7 @@ class QuerySpace extends refined_space_1.RefinedSpace {
             const newExtends = [];
             const extendedStruct = this.exprSpace.structDef();
             for (const extendName of this.exprSpace.extendList) {
-                const extendEnt = extendedStruct.fields.find(f => (0, field_utils_1.nameOf)(f) === extendName);
+                const extendEnt = extendedStruct.fields.find(f => (0, field_utils_1.nameFromDef)(f) === extendName);
                 if (extendEnt) {
                     newExtends.push(extendEnt);
                 }
@@ -245,12 +257,6 @@ class QuerySpace extends refined_space_1.RefinedSpace {
     isQueryFieldSpace() {
         return true;
     }
-    outputSpace() {
-        return this;
-    }
-    inputSpace() {
-        return this.exprSpace;
-    }
 }
 exports.QuerySpace = QuerySpace;
 class ReduceFieldSpace extends QuerySpace {
@@ -261,9 +267,6 @@ class ReduceFieldSpace extends QuerySpace {
 }
 exports.ReduceFieldSpace = ReduceFieldSpace;
 function isEmptyNest(fd) {
-    return (typeof fd !== 'string' &&
-        !model.isFilteredAliasedName(fd) &&
-        fd.type === 'turtle' &&
-        fd.pipeline.length === 0);
+    return (typeof fd !== 'string' && fd.type === 'turtle' && fd.pipeline.length === 0);
 }
 //# sourceMappingURL=query-spaces.js.map