@malloydata/malloy 0.0.104-dev231116200719 → 0.0.104-dev231117214047

package/dist/lang/ast/elements/pipeline-desc.d.ts

@@ -2,7 +2,7 @@ import { Annotation, DocumentLocation, PipeSegment, Pipeline, StructDef } from '
 import { FieldSpace } from '../types/field-space';
 import { MalloyElement } from '../types/malloy-element';
 import { QOPDesc } from '../query-properties/qop-desc';
-import { QueryInputSpace } from '../field-space/query-input-space';
+import { QuerySpace } from '../field-space/query-spaces';
 import { ViewFieldReference } from '../query-items/field-references';
 import { Refinement } from '../query-properties/refinements';
 interface AppendResult {
@@ -17,7 +17,18 @@ interface AppendResult {
 export declare abstract class PipelineDesc extends MalloyElement {
     protected refinements?: Refinement[];
     protected qops: QOPDesc[];
-    nestedInQuerySpace?: QueryInputSpace;
+    private isNestIn?;
+    /**
+     * This pipeline is actually a nest statement, and the passed query space
+     * is the space for the query which contains the nest statement. This is
+     * used so that nest queries can walk up a nest chain to check
+     * "ungrouping" expressions.
+     *
+     * This is only here so that it can be used when a Builder is created
+     * so the query space created by the builder can also know that it is
+     * nested.
+     */
+    declareAsNestInside(qs: QuerySpace): void;
     alreadyRefined(): boolean;
     refineWith(refinements: (QOPDesc | ViewFieldReference)[]): void;
     addSegments(...segDesc: QOPDesc[]): void;

package/dist/lang/ast/elements/pipeline-desc.js

@@ -40,6 +40,19 @@ class PipelineDesc extends malloy_element_1.MalloyElement {
         super(...arguments);
         this.qops = [];
     }
+    /**
+     * This pipeline is actually a nest statement, and the passed query space
+     * is the space for the query which contains the nest statement. This is
+     * used so that nest queries can walk up a nest chain to check
+     * "ungrouping" expressions.
+     *
+     * This is only here so that it can be used when a Builder is created
+     * so the query space created by the builder can also know that it is
+     * nested.
+     */
+    declareAsNestInside(qs) {
+        this.isNestIn = qs;
+    }
     alreadyRefined() {
         return this.refinements !== undefined;
     }
@@ -54,7 +67,7 @@ class PipelineDesc extends malloy_element_1.MalloyElement {
     }
     appendOps(pipelineOutput, modelPipe) {
         const returnPipe = [...modelPipe];
-        const nestedIn = modelPipe.length === 0 ? this.nestedInQuerySpace : undefined;
+        const nestedIn = modelPipe.length === 0 ? this.isNestIn : undefined;
         let nextFS = () => pipelineOutput;
         for (const qop of this.qops) {
             const next = qop.getOp(nextFS(), nestedIn);
@@ -81,7 +94,7 @@ class PipelineDesc extends malloy_element_1.MalloyElement {
         }
         pipeline.push(...modelPipe.pipeline);
         for (const refinement of this.refinements) {
-            pipeline = refinement.refine(fs, pipeline);
+            pipeline = refinement.refine(fs, pipeline, this.isNestIn);
         }
         return { pipeline };
     }

package/dist/lang/ast/expressions/constant-sub-expression.js

@@ -52,9 +52,6 @@ class ConstantFieldSpace {
     dialectObj() {
         return undefined;
     }
-    whenComplete(step) {
-        step();
-    }
     isQueryFieldSpace() {
         return false;
     }

package/dist/lang/ast/expressions/expr-ungroup.js

@@ -55,28 +55,33 @@ class ExprUngroup extends expression_def_1.ExpressionDef {
             e: exprVal.value,
         };
         if (this.typeCheck(this.expr, { ...exprVal, expressionType: 'scalar' })) {
-            if (this.fields.length > 0) {
-                if (!fs.isQueryFieldSpace()) {
-                    this.log(`${this.control}() must be in a query -- weird internal error`);
-                    return (0, ast_utils_1.errorFor)('ungroup query check');
-                }
-                const output = fs.outputSpace();
-                if (!(output instanceof query_spaces_1.QuerySpace)) {
-                    // TODO maybe make OutputSpace return an interface which has checkUngroup
-                    this.log(`${this.control}() must be in a query -- weird internal error`);
-                    return (0, ast_utils_1.errorFor)('ungroup query check');
-                }
+            // Now every mentioned field must be in the output space of one of the queries
+            // of the nest tree leading to this query. If this is a source definition,
+            // this is not checked until sql generation time.
+            if (fs.isQueryFieldSpace() && this.fields.length > 0) {
                 const dstFields = [];
                 const isExclude = this.control === 'exclude';
-                for (const mustBeInOutput of this.fields) {
-                    output.whenComplete(() => {
-                        output.checkUngroup(mustBeInOutput, isExclude);
-                    });
-                    dstFields.push(mustBeInOutput.refString);
+                for (const mentionedField of this.fields) {
+                    let ofs = fs.outputSpace();
+                    let notFound = true;
+                    while (ofs) {
+                        const entryInfo = ofs.lookup([mentionedField]);
+                        if (entryInfo.found && entryInfo.isOutputField) {
+                            dstFields.push(mentionedField.refString);
+                            notFound = false;
+                        }
+                        else if (ofs instanceof query_spaces_1.QuerySpace) {
+                            // should always be true, but don't have types right, thus the if
+                            ofs = ofs.nestParent;
+                            continue;
+                        }
+                        break;
+                    }
+                    if (notFound) {
+                        const uName = isExclude ? 'exclude()' : 'all()';
+                        mentionedField.log(`${uName} '${mentionedField.refString}' is missing from query output`);
+                    }
                 }
-                // TODO maybe now we can just look up the fields in the output space now and ensure
-                // they're all there, rather than waiting until the query is finished to do it?
-                // See `order_by` for an example of how this could work.
                 ungroup.fields = dstFields;
             }
             return {

package/dist/lang/ast/field-space/dynamic-space.d.ts

@@ -11,7 +11,6 @@ export declare abstract class DynamicSpace extends StaticSpace {
     private complete;
     protected newTimezone?: string;
     constructor(extending: SourceSpec);
-    whenComplete(finalizeStep: () => void): void;
     isComplete(): void;
     protected setEntry(name: string, value: SpaceEntry): void;
     addParameters(params: HasParameter[]): DynamicSpace;

package/dist/lang/ast/field-space/dynamic-space.js

@@ -45,20 +45,8 @@ class DynamicSpace extends static_space_1.StaticSpace {
         this.final = undefined;
         this.source = source;
     }
-    whenComplete(finalizeStep) {
-        if (this.complete) {
-            finalizeStep();
-        }
-        else {
-            this.completions.push(finalizeStep);
-        }
-    }
     isComplete() {
         this.complete = true;
-        for (const step of this.completions) {
-            step();
-        }
-        this.completions = [];
     }
     setEntry(name, value) {
         if (this.final) {

package/dist/lang/ast/field-space/query-input-space.d.ts

@@ -12,7 +12,6 @@ import { FieldSpace, QueryFieldSpace } from '../types/field-space';
 import { RefinedSpace } from './refined-space';
 export declare class QueryInputSpace extends RefinedSpace implements QueryFieldSpace {
     private queryOutput;
-    nestParent?: QueryInputSpace;
     extendList: string[];
     /**
      * Because of circularity concerns this constructor is not typed

package/dist/lang/ast/field-space/query-spaces.d.ts

@@ -17,18 +17,12 @@ export declare abstract class QuerySpace extends RefinedSpace implements QueryFi
     astEl?: MalloyElement | undefined;
     abstract readonly segmentType: 'reduce' | 'project' | 'index';
     expandedWild: Record<string, string[]>;
+    nestParent?: QuerySpace;
     constructor(queryInputSpace: FieldSpace, refineThis: model.PipeSegment | undefined);
     private addRefineFromFields;
     log(s: string): void;
     pushFields(...defs: MalloyElement[]): void;
     protected addWild(wild: WildcardFieldReference): void;
-    /**
-     * Check for the definition of an ungrouping reference in the result space,
-     * or in the case of an exclude reference, if this query is nested
-     * in another query, in the result space of a query that this query
-     * is nested inside of.
-     */
-    checkUngroup(fn: FieldName, isExclude: boolean): void;
     canContain(_typeDesc: model.TypeDesc): boolean;
     protected queryFieldDefs(): model.QueryFieldDef[];
     getQuerySegment(rf: model.QuerySegment | undefined): model.QuerySegment;

package/dist/lang/ast/field-space/query-spaces.js

@@ -155,33 +155,6 @@ class QuerySpace extends refined_space_1.RefinedSpace {
             }
         }
     }
-    /**
-     * Check for the definition of an ungrouping reference in the result space,
-     * or in the case of an exclude reference, if this query is nested
-     * in another query, in the result space of a query that this query
-     * is nested inside of.
-     */
-    checkUngroup(fn, isExclude) {
-        if (!this.entry(fn.refString)) {
-            const parent = this.exprSpace.nestParent;
-            if (isExclude && parent) {
-                parent.whenComplete(() => {
-                    const pOut = parent.outputSpace();
-                    // a little ugly, but it breaks a circularity problem
-                    if (pOut instanceof QuerySpace) {
-                        pOut.checkUngroup(fn, isExclude);
-                    }
-                    else {
-                        throw new Error('OUCH');
-                    }
-                });
-            }
-            else {
-                const uName = isExclude ? 'exclude()' : 'all()';
-                fn.log(`${uName} '${fn.refString}' is missing from query output`);
-            }
-        }
-    }
     canContain(_typeDesc) {
         return true;
     }

package/dist/lang/ast/field-space/static-space.d.ts

@@ -10,7 +10,6 @@ export declare class StaticSpace implements FieldSpace {
     private memoMap?;
     protected fromStruct: StructDef;
     constructor(sourceStructDef: StructDef);
-    whenComplete(step: () => void): void;
     dialectObj(): Dialect | undefined;
     defToSpaceField(from: FieldDef): SpaceField;
     private get map();

package/dist/lang/ast/field-space/static-space.js

@@ -35,9 +35,6 @@ class StaticSpace {
         this.type = 'fieldSpace';
         this.fromStruct = sourceStructDef;
     }
-    whenComplete(step) {
-        step();
-    }
     dialectObj() {
         try {
             return (0, dialect_map_1.getDialect)(this.fromStruct.dialect);
@@ -104,22 +101,6 @@ class StaticSpace {
             return { error: `'${head}' is not defined`, found };
         }
         if (found instanceof space_field_1.SpaceField) {
-            /*
-             * TODO cache defs, post the addReference call to whenComplete
-             *
-             * In the cleanup phase of query space construction, it may check
-             * the output space for ungrouping variables. However if an
-             * ungrouping variable is a measure, the field expression value
-             * needed to get the definition needs to be computed in the
-             * input space of the query. There is a test which failed which
-             * caused this code to be here, but this is really a bandaid.
-             *
-             * Some re-work of how to get the definition of a SpaceField
-             * no matter what it is contained in needs to be thought out.
-             *
-             * ... or this check would look at the finalized output of
-             * the namespace and not re-compile ...
-             */
             const definition = found.fieldDef();
             if (definition) {
                 head.addReference({

package/dist/lang/ast/query-items/field-declaration.d.ts

@@ -68,7 +68,6 @@ export declare class DefSpace implements FieldSpace {
     lookup(symbol: FieldName[]): LookupResult;
     entries(): [string, SpaceEntry][];
     dialectObj(): Dialect | undefined;
-    whenComplete(step: () => void): void;
     isQueryFieldSpace(): this is QueryFieldSpace;
     outputSpace(): FieldSpace;
     inputSpace(): FieldSpace;

package/dist/lang/ast/query-items/field-declaration.js

@@ -226,9 +226,6 @@ class DefSpace {
     dialectObj() {
         return this.realFS.dialectObj();
     }
-    whenComplete(step) {
-        this.realFS.whenComplete(step);
-    }
     isQueryFieldSpace() {
         return this.realFS.isQueryFieldSpace();
     }

package/dist/lang/ast/query-properties/nest.d.ts

@@ -4,11 +4,11 @@ import { MalloyElement } from '../types/malloy-element';
 import { Noteable, extendNoteMethod } from '../types/noteable';
 import { QueryField } from '../field-space/query-space-field';
 import { TurtleHeadedPipe } from '../elements/pipeline-desc';
-import { QueryInputSpace } from '../field-space/query-input-space';
 import { LegalRefinementStage, QueryClass, QueryPropertyInterface } from '../types/query-property-interface';
 import { QueryBuilder } from '../types/query-builder';
 import { MakeEntry } from '../types/space-entry';
 import { DynamicSpace } from '../field-space/dynamic-space';
+import { QuerySpace } from '../field-space/query-spaces';
 import { TypeDesc } from '../../../model';
 import { FieldReference, ViewFieldReference } from '../query-items/field-references';
 declare abstract class TurtleDeclRoot extends TurtleHeadedPipe implements Noteable, MakeEntry {
@@ -18,7 +18,7 @@ declare abstract class TurtleDeclRoot extends TurtleHeadedPipe implements Noteab
     note?: model.Annotation;
     constructor(name: string);
     getPipeline(fs: FieldSpace): model.Pipeline;
-    getFieldDef(fs: FieldSpace, nestParent: QueryInputSpace | undefined): model.TurtleDef;
+    getFieldDef(fs: FieldSpace): model.TurtleDef;
     makeEntry(fs: DynamicSpace): void;
 }
 export declare class TurtleDecl extends TurtleDeclRoot {
@@ -41,15 +41,17 @@ export declare class NestDefinition extends TurtleDeclRoot implements QueryPrope
     makeEntry(fs: DynamicSpace): void;
 }
 export declare function isNestedQuery(me: MalloyElement): me is NestedQuery;
-export declare class QueryFieldAST extends QueryField {
+export declare class ViewField extends QueryField {
     readonly turtle: TurtleDecl;
     protected name: string;
     renameAs?: string;
-    nestParent?: QueryInputSpace;
     constructor(fs: FieldSpace, turtle: TurtleDecl, name: string);
     getQueryFieldDef(fs: FieldSpace): model.QueryFieldDef;
     fieldDef(): model.TurtleDef;
 }
+export declare class NestField extends ViewField {
+    constructor(fs: FieldSpace, turtle: TurtleDecl, name: string, spaceContainingNest: QuerySpace);
+}
 export declare class NestReference extends FieldReference implements QueryPropertyInterface, MakeEntry {
     readonly name: FieldReference;
     elementType: string;

package/dist/lang/ast/query-properties/nest.js

@@ -45,7 +45,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.NestReference = exports.QueryFieldAST = exports.isNestedQuery = exports.NestDefinition = exports.NestRefinement = exports.TurtleDecl = void 0;
+exports.NestReference = exports.NestField = exports.ViewField = exports.isNestedQuery = exports.NestDefinition = exports.NestRefinement = exports.TurtleDecl = void 0;
 const model = __importStar(require("../../../model/malloy_types"));
 const noteable_1 = require("../types/noteable");
 const query_space_field_1 = require("../field-space/query-space-field");
@@ -126,10 +126,7 @@ class TurtleDeclRoot extends pipeline_desc_1.TurtleHeadedPipe {
         }
         return modelPipe;
     }
-    getFieldDef(fs, nestParent) {
-        if (nestParent) {
-            this.nestedInQuerySpace = nestParent;
-        }
+    getFieldDef(fs) {
         const pipe = this.getPipeline(fs);
         const turtle = {
             type: 'turtle',
@@ -148,7 +145,7 @@ class TurtleDeclRoot extends pipeline_desc_1.TurtleHeadedPipe {
         return turtle;
     }
     makeEntry(fs) {
-        fs.newEntry(this.name, this, new QueryFieldAST(fs, this, this.name));
+        fs.newEntry(this.name, this, new ViewField(fs, this, this.name));
     }
 }
 class TurtleDecl extends TurtleDeclRoot {
@@ -171,8 +168,7 @@ class NestRefinement extends TurtleDeclRoot {
     }
     makeEntry(fs) {
         if (fs instanceof query_spaces_1.QuerySpace) {
-            const qf = new QueryFieldAST(fs, this, this.name);
-            qf.nestParent = fs.inputSpace();
+            const qf = new NestField(fs, this, this.name, fs);
             fs.newEntry(this.name, this, qf);
             return;
         }
@@ -192,8 +188,7 @@ class NestDefinition extends TurtleDeclRoot {
     }
     makeEntry(fs) {
         if (fs instanceof query_spaces_1.QuerySpace) {
-            const qf = new QueryFieldAST(fs, this, this.name);
-            qf.nestParent = fs.inputSpace();
+            const qf = new NestField(fs, this, this.name, fs);
             fs.newEntry(this.name, this, qf);
             return;
         }
@@ -207,28 +202,35 @@ function isNestedQuery(me) {
         me instanceof NestDefinition);
 }
 exports.isNestedQuery = isNestedQuery;
-class QueryFieldAST extends query_space_field_1.QueryField {
+class ViewField extends query_space_field_1.QueryField {
     constructor(fs, turtle, name) {
         super(fs);
         this.turtle = turtle;
         this.name = name;
     }
     getQueryFieldDef(fs) {
-        const def = this.turtle.getFieldDef(fs, this.nestParent);
+        const def = this.turtle.getFieldDef(fs);
         if (this.renameAs) {
             def.as = this.renameAs;
         }
         return def;
     }
     fieldDef() {
-        const def = this.turtle.getFieldDef(this.inSpace, this.nestParent);
+        const def = this.turtle.getFieldDef(this.inSpace);
         if (this.renameAs) {
             def.as = this.renameAs;
         }
         return def;
     }
 }
-exports.QueryFieldAST = QueryFieldAST;
+exports.ViewField = ViewField;
+class NestField extends ViewField {
+    constructor(fs, turtle, name, spaceContainingNest) {
+        super(fs, turtle, name);
+        turtle.declareAsNestInside(spaceContainingNest);
+    }
+}
+exports.NestField = NestField;
 class NestReference extends field_references_1.FieldReference {
     constructor(name) {
         super([...name.list]);

package/dist/lang/ast/query-properties/qop-desc.d.ts

@@ -4,7 +4,7 @@ import { ListOf } from '../types/malloy-element';
 import { OpDesc } from '../types/op-desc';
 import { QueryProperty } from '../types/query-property';
 import { QueryClass } from '../types/query-property-interface';
-import { QueryInputSpace } from '../field-space/query-input-space';
+import { QuerySpace } from '../field-space/query-spaces';
 export declare class QOPDesc extends ListOf<QueryProperty> {
     elementType: string;
     opClass: QueryClass | undefined;
@@ -12,5 +12,5 @@ export declare class QOPDesc extends ListOf<QueryProperty> {
     protected computeType(): QueryClass | undefined;
     refineFrom(existing: PipeSegment): void;
     private getBuilder;
-    getOp(inputFS: FieldSpace, headFieldSpace: QueryInputSpace | undefined): OpDesc;
+    getOp(inputFS: FieldSpace, isNestIn: QuerySpace | undefined): OpDesc;
 }

package/dist/lang/ast/query-properties/qop-desc.js

@@ -86,10 +86,10 @@ class QOPDesc extends malloy_element_1.ListOf {
                 return new partial_builder_1.PartialBuilder(baseFS, this.refineThis);
         }
     }
-    getOp(inputFS, headFieldSpace) {
+    getOp(inputFS, isNestIn) {
         const build = this.getBuilder(inputFS);
-        if (headFieldSpace) {
-            build.inputFS.nestParent = headFieldSpace;
+        if (isNestIn) {
+            build.resultFS.nestParent = isNestIn;
         }
         build.resultFS.astEl = this;
         for (const qp of this.list) {

package/dist/lang/ast/query-properties/refinements.d.ts

@@ -4,8 +4,14 @@ import { MalloyElement } from '../types/malloy-element';
 import { OpDesc } from '../types/op-desc';
 import { ViewFieldReference } from '../query-items/field-references';
 import { QOPDesc } from './qop-desc';
+import { QuerySpace } from '../field-space/query-spaces';
 export declare abstract class Refinement extends MalloyElement {
-    abstract refine(inputFS: FieldSpace, pipeline: PipeSegment[]): PipeSegment[];
+    /**
+     * @param inputFS
+     * @param pipeline
+     * @param isNestIn The pipeline being refined is a nest, and this is the space which contains the nest statement
+     */
+    abstract refine(inputFS: FieldSpace, pipeline: PipeSegment[], isNestIn: QuerySpace | undefined): PipeSegment[];
     static from(base: QOPDesc | ViewFieldReference): QOPDescRefinement | NamedRefinement;
 }
 export declare class NamedRefinement extends Refinement {
@@ -13,13 +19,13 @@ export declare class NamedRefinement extends Refinement {
     elementType: string;
     constructor(name: ViewFieldReference);
     private getRefinementSegment;
-    refine(inputFS: FieldSpace, pipeline: PipeSegment[]): PipeSegment[];
-    getOp(inputFS: FieldSpace, _to: PipeSegment): OpDesc;
+    refine(inputFS: FieldSpace, pipeline: PipeSegment[], _isNestIn: QuerySpace | undefined): PipeSegment[];
+    getOp(inputFS: FieldSpace, refineTo: PipeSegment): OpDesc;
 }
 export declare class QOPDescRefinement extends Refinement {
     private readonly qOpDesc;
     elementType: string;
     constructor(qOpDesc: QOPDesc);
     private getOp;
-    refine(inputFS: FieldSpace, _pipeline: PipeSegment[]): PipeSegment[];
+    refine(inputFS: FieldSpace, _pipeline: PipeSegment[], isNestIn: QuerySpace | undefined): PipeSegment[];
 }

package/dist/lang/ast/query-properties/refinements.js

@@ -70,7 +70,7 @@ class NamedRefinement extends Refinement {
         }
         this.name.log(`named refinement \`${this.name.refString}\` must be a view, found a ${res.found.typeDesc().dataType}`);
     }
-    refine(inputFS, pipeline) {
+    refine(inputFS, pipeline, _isNestIn) {
         if (pipeline.length === 1) {
             return [this.getOp(inputFS, pipeline[0]).segment];
         }
@@ -80,9 +80,9 @@ class NamedRefinement extends Refinement {
             return pipeline;
         }
     }
-    getOp(inputFS, _to) {
+    getOp(inputFS, refineTo) {
         var _a, _b;
-        const to = { ..._to };
+        const to = { ...refineTo };
         const from = this.getRefinementSegment(inputFS);
         if (from) {
             // TODO need to disallow partial + index for now to make the types happy
@@ -155,15 +155,15 @@ class QOPDescRefinement extends Refinement {
         this.qOpDesc = qOpDesc;
         this.elementType = 'qopdescRefinement';
     }
-    getOp(inputFS, headFS, qOpDesc, refineThis) {
+    getOp(inputFS, isNestIn, qOpDesc, refineThis) {
         qOpDesc.refineFrom(refineThis);
-        return qOpDesc.getOp(inputFS, headFS).segment;
+        return qOpDesc.getOp(inputFS, isNestIn).segment;
     }
-    refine(inputFS, _pipeline) {
+    refine(inputFS, _pipeline, isNestIn) {
         const pipeline = [..._pipeline];
         if (pipeline.length === 1) {
             this.qOpDesc.refineFrom(pipeline[0]);
-            return [this.getOp(inputFS, undefined, this.qOpDesc, pipeline[0])];
+            return [this.getOp(inputFS, isNestIn, this.qOpDesc, pipeline[0])];
         }
         const headRefinements = new qop_desc_1.QOPDesc([]);
         const tailRefinements = new qop_desc_1.QOPDesc([]);

package/dist/lang/ast/types/field-space.d.ts

@@ -15,7 +15,6 @@ export interface FieldSpace {
     entry(symbol: string): SpaceEntry | undefined;
     entries(): [string, SpaceEntry][];
     dialectObj(): Dialect | undefined;
-    whenComplete: (step: () => void) => void;
     isQueryFieldSpace(): this is QueryFieldSpace;
 }
 export interface QueryFieldSpace extends FieldSpace {

package/dist/lang/parse-malloy.d.ts

@@ -3,7 +3,7 @@ import { DocumentLocation, DocumentPosition, DocumentRange, DocumentReference, I
 import { LogMessage, MessageLog, MessageLogger } from './parse-log';
 import { Zone, ZoneData } from './zone';
 import { ReferenceList } from './reference-list';
-import { ASTResponse, CompletionsResponse, DataRequestResponse, ProblemResponse, FatalResponse, FinalResponse, HelpContextResponse, MetadataResponse, ModelDataRequest, NeedURLData, TranslateResponse } from './translate-response';
+import { ASTResponse, CompletionsResponse, DataRequestResponse, ProblemResponse, FatalResponse, FinalResponse, HelpContextResponse, MetadataResponse, ModelDataRequest, NeedURLData, TranslateResponse, ModelAnnotationResponse } from './translate-response';
 import { Tag } from '../tags';
 import { MalloyParseInfo } from './malloy-parse-info';
 export type StepResponses = DataRequestResponse | ASTResponse | TranslateResponse | ParseResponse | MetadataResponse;
@@ -83,6 +83,12 @@ declare class HelpContextStep implements TranslationStep {
         character: number;
     }): HelpContextResponse;
 }
+declare class ModelAnnotationStep implements TranslationStep {
+    readonly parseStep: ParseStep;
+    response?: ModelAnnotationResponse;
+    constructor(parseStep: ParseStep);
+    step(that: MalloyTranslation, extendingModel?: ModelDef): ModelAnnotationResponse;
+}
 declare class TranslateStep implements TranslationStep {
     readonly astStep: ASTStep;
     response?: TranslateResponse;
@@ -103,6 +109,7 @@ export declare abstract class MalloyTranslation {
     imports: ImportLocation[];
     compilerFlags: Tag;
     readonly parseStep: ParseStep;
+    readonly modelAnnotationStep: ModelAnnotationStep;
     readonly importsAndTablesStep: ImportsAndTablesStep;
     readonly astStep: ASTStep;
     readonly metadataStep: MetadataStep;
@@ -136,6 +143,7 @@ export declare abstract class MalloyTranslation {
     translate(extendingModel?: ModelDef): TranslateResponse;
     importAt(position: DocumentPosition): ImportLocation | undefined;
     metadata(): MetadataResponse;
+    modelAnnotation(extendingModel?: ModelDef): ModelAnnotationResponse;
     completions(position: {
         line: number;
         character: number;

package/dist/lang/parse-malloy.js

@@ -62,6 +62,7 @@ const reference_list_1 = require("./reference-list");
 const translate_response_1 = require("./translate-response");
 const utils_1 = require("./utils");
 const tags_1 = require("../tags");
+const model_annotation_walker_1 = require("./parse-tree-walkers/model-annotation-walker");
 /**
  * This ignores a -> popMode when the mode stack is empty, which is a hack,
  * but it let's us parse }%
@@ -451,6 +452,29 @@ class HelpContextStep {
         }
     }
 }
+class ModelAnnotationStep {
+    constructor(parseStep) {
+        this.parseStep = parseStep;
+    }
+    step(that, extendingModel) {
+        if (!this.response) {
+            const tryParse = this.parseStep.step(that);
+            if (!tryParse.parse || tryParse.final) {
+                return tryParse;
+            }
+            else {
+                const modelAnnotation = (0, model_annotation_walker_1.walkForModelAnnotation)(that, tryParse.parse.tokenStream, tryParse.parse);
+                this.response = {
+                    modelAnnotation: {
+                        ...modelAnnotation,
+                        inherits: extendingModel === null || extendingModel === void 0 ? void 0 : extendingModel.annotation,
+                    },
+                };
+            }
+        }
+        return this.response;
+    }
+}
 class TranslateStep {
     constructor(astStep) {
         this.astStep = astStep;
@@ -545,6 +569,7 @@ class MalloyTranslation {
          * things will happen automatically.
          */
         this.parseStep = new ParseStep();
+        this.modelAnnotationStep = new ModelAnnotationStep(this.parseStep);
         this.metadataStep = new MetadataStep(this.parseStep);
         this.completionsStep = new CompletionsStep(this.parseStep);
         this.helpContextStep = new HelpContextStep(this.parseStep);
@@ -703,6 +728,9 @@ class MalloyTranslation {
     metadata() {
         return this.metadataStep.step(this);
     }
+    modelAnnotation(extendingModel) {
+        return this.modelAnnotationStep.step(this, extendingModel);
+    }
     completions(position) {
         return this.completionsStep.step(this, position);
     }

package/dist/lang/parse-tree-walkers/model-annotation-walker.d.ts

@@ -0,0 +1,5 @@
+import { CommonTokenStream } from 'antlr4ts';
+import { MalloyTranslation } from '../parse-malloy';
+import { Annotation } from '../../model/malloy_types';
+import { MalloyParseInfo } from '../malloy-parse-info';
+export declare function walkForModelAnnotation(forParse: MalloyTranslation, tokens: CommonTokenStream, parseInfo: MalloyParseInfo): Annotation;

package/dist/lang/parse-tree-walkers/model-annotation-walker.js

@@ -0,0 +1,60 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.walkForModelAnnotation = void 0;
+const ParseTreeWalker_1 = require("antlr4ts/tree/ParseTreeWalker");
+class ModelAnnotationWalker {
+    constructor(translator, tokens, parseInfo) {
+        this.translator = translator;
+        this.tokens = tokens;
+        this.parseInfo = parseInfo;
+        this.notes = [];
+    }
+    getLocation(cx) {
+        return {
+            url: this.parseInfo.sourceURL,
+            range: this.parseInfo.rangeFromContext(cx),
+        };
+    }
+    enterDocAnnotations(pcx) {
+        const allNotes = pcx.DOC_ANNOTATION().map(note => {
+            return {
+                text: note.text,
+                at: this.getLocation(pcx),
+            };
+        });
+        this.notes.push(...allNotes);
+    }
+    get annotation() {
+        return { notes: this.notes };
+    }
+}
+function walkForModelAnnotation(forParse, tokens, parseInfo) {
+    const finder = new ModelAnnotationWalker(forParse, tokens, parseInfo);
+    const listener = finder;
+    ParseTreeWalker_1.ParseTreeWalker.DEFAULT.walk(listener, parseInfo.root);
+    return finder.annotation;
+}
+exports.walkForModelAnnotation = walkForModelAnnotation;
+//# sourceMappingURL=model-annotation-walker.js.map

package/dist/lang/test/model-annotation-walker.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lang/test/model-annotation-walker.spec.js

@@ -0,0 +1,46 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_translator_1 = require("./test-translator");
+test('model annotations can be retrieved', () => {
+    var _a, _b;
+    const src = (0, test_translator_1.markSource) `${'## foo'}`;
+    const doc = new test_translator_1.TestTranslator(src.code);
+    const { modelAnnotation } = doc.modelAnnotation();
+    expect((_a = modelAnnotation === null || modelAnnotation === void 0 ? void 0 : modelAnnotation.notes) === null || _a === void 0 ? void 0 : _a.length).toBe(1);
+    const notes = (_b = modelAnnotation === null || modelAnnotation === void 0 ? void 0 : modelAnnotation.notes) !== null && _b !== void 0 ? _b : [];
+    expect(notes[0].text).toBe('## foo');
+    expect(notes[0].at).toMatchObject(src.locations[0]);
+});
+test('does not explode if bad parse', () => {
+    const src = (0, test_translator_1.markSource) `
+    ${'## foo'}
+    asd afsjei ; duavby {{}}}
+  `;
+    const doc = new test_translator_1.TestTranslator(src.code);
+    const response = doc.modelAnnotation();
+    expect(response.modelAnnotation).toBe(undefined);
+    expect(response.final).toBe(true);
+});
+//# sourceMappingURL=model-annotation-walker.spec.js.map

package/dist/lang/test/query.spec.js

@@ -187,6 +187,26 @@ describe('query:', () => {
       }
     `).toTranslate();
         });
+        test('exclude output checking survives refinement', () => {
+            // This was https://github.com/malloydata/malloy/issues/1474
+            const nestExclude = (0, test_translator_1.model) `
+        source: flights is a extend {
+          dimension: carrier is astr, destination is astr
+          measure: flight_count is count()
+          view: by_dest is {
+            group_by: destination
+            aggregate: flight_count
+          }
+        }
+        run: flights -> {
+          group_by: carrier
+          nest: broken is by_dest + {
+            top: 5
+            aggregate: flights_to_dest_all_carriers is exclude(flight_count, carrier)
+          }
+        }`;
+            expect(nestExclude).toTranslate();
+        });
     });
     describe('query operation typechecking', () => {
         describe('field declarations', () => {

package/dist/lang/translate-response.d.ts

@@ -1,4 +1,4 @@
-import { ModelDef, Query, SQLBlockSource, SQLBlockStructDef } from '../model/malloy_types';
+import { Annotation, ModelDef, Query, SQLBlockSource, SQLBlockStructDef } from '../model/malloy_types';
 import { MalloyElement } from './ast';
 import { LogMessage } from './parse-log';
 import { DocumentSymbol } from './parse-tree-walkers/document-symbol-walker';
@@ -43,6 +43,10 @@ interface Metadata extends NeededData, ProblemResponse, FinalResponse {
     highlights: DocumentHighlight[];
 }
 export type MetadataResponse = Partial<Metadata>;
+interface ModelAnnotationData extends NeededData, ProblemResponse, FinalResponse {
+    modelAnnotation: Annotation;
+}
+export type ModelAnnotationResponse = Partial<ModelAnnotationData>;
 interface Completions extends NeededData, ProblemResponse, FinalResponse {
     completions: DocumentCompletion[];
 }

package/dist/malloy.d.ts

@@ -2,7 +2,7 @@
 import { RunSQLOptions } from './run_sql_options';
 import { DocumentCompletion as DocumentCompletionDefinition, DocumentHighlight as DocumentHighlightDefinition, DocumentSymbol as DocumentSymbolDefinition, LogMessage, MalloyTranslator } from './lang';
 import { DocumentHelpContext } from './lang/parse-tree-walkers/document-help-context-walker';
-import { CompiledQuery, DocumentLocation, DocumentReference, FieldBooleanDef, FieldDateDef, FieldJSONDef, FieldNumberDef, FieldStringDef, FieldTimestampDef, FieldTypeDef, FilterExpression, Query as InternalQuery, ModelDef, DocumentPosition as ModelDocumentPosition, NamedQuery, QueryData, QueryDataRow, QueryResult, SQLBlock, SQLBlockStructDef, SearchIndexResult, SearchValueMapResult, StructDef, TurtleDef, FieldUnsupportedDef, QueryRunStats, ImportLocation } from './model';
+import { CompiledQuery, DocumentLocation, DocumentReference, FieldBooleanDef, FieldDateDef, FieldJSONDef, FieldNumberDef, FieldStringDef, FieldTimestampDef, FieldTypeDef, FilterExpression, Query as InternalQuery, ModelDef, DocumentPosition as ModelDocumentPosition, NamedQuery, QueryData, QueryDataRow, QueryResult, SQLBlock, SQLBlockStructDef, SearchIndexResult, SearchValueMapResult, StructDef, TurtleDef, FieldUnsupportedDef, QueryRunStats, ImportLocation, Annotation } from './model';
 import { Connection, InfoConnection, LookupConnection, ModelString, ModelURL, QueryString, QueryURL, URLReader } from './runtime_types';
 import { Tag, TagParse, TagParseSpec, Taggable } from './tags';
 export interface Loggable {
@@ -425,6 +425,8 @@ export declare class PreparedResult implements Taggable {
     constructor(query: CompiledQuery, modelDef: ModelDef);
     tagParse(spec?: TagParseSpec): TagParse;
     getTaglines(prefix?: RegExp): string[];
+    get annotation(): Annotation | undefined;
+    get modelAnnotation(): Annotation | undefined;
     /**
      * @return The name of the connection this query should be run against.
      */

package/dist/malloy.js

@@ -129,6 +129,7 @@ class Malloy {
                         }
                     }
                 }
+                const { modelAnnotation } = translator.modelAnnotation(model === null || model === void 0 ? void 0 : model._modelDef);
                 if (result.tables) {
                     // collect tables by connection name since there may be multiple connections
                     const tablesByConnection = new Map();
@@ -151,6 +152,7 @@ class Malloy {
                             //      the translator runs into an infinite loop fetching tables.
                             const { schemas: tables, errors } = await connection.fetchSchemaForTables(tablePathByKey, {
                                 refreshTimestamp,
+                                modelAnnotation,
                             });
                             translator.update({ tables, errors: { tables: errors } });
                         }
@@ -175,6 +177,7 @@ class Malloy {
                         const expanded = Malloy.compileSQLBlock(result.partialModel, toCompile);
                         const resolved = await conn.fetchSchemaForSQLBlock(expanded, {
                             refreshTimestamp,
+                            modelAnnotation,
                         });
                         if (resolved.error) {
                             translator.update({
@@ -747,6 +750,12 @@ class PreparedResult {
     getTaglines(prefix) {
         return tags_1.Tag.annotationToTaglines(this.inner.annotation, prefix);
     }
+    get annotation() {
+        return this.inner.annotation;
+    }
+    get modelAnnotation() {
+        return this.modelDef.annotation;
+    }
     /**
      * @return The name of the connection this query should be run against.
      */
@@ -2033,12 +2042,18 @@ class QueryMaterializer extends FluentState {
     async run(options) {
         const connections = this.runtime.connections;
         const preparedResult = await this.getPreparedResult();
-        return Malloy.run({ connections, preparedResult, options });
+        const finalOptions = runSQLOptionsWithAnnotations(preparedResult, options);
+        return Malloy.run({ connections, preparedResult, options: finalOptions });
     }
     async *runStream(options) {
         const preparedResult = await this.getPreparedResult();
         const connections = this.runtime.connections;
-        const stream = Malloy.runStream({ connections, preparedResult, options });
+        const finalOptions = runSQLOptionsWithAnnotations(preparedResult, options);
+        const stream = Malloy.runStream({
+            connections,
+            preparedResult,
+            options: finalOptions,
+        });
         for await (const row of stream) {
             yield row;
         }
@@ -2090,6 +2105,13 @@ class QueryMaterializer extends FluentState {
     }
 }
 exports.QueryMaterializer = QueryMaterializer;
+function runSQLOptionsWithAnnotations(preparedResult, givenOptions) {
+    return {
+        queryAnnotation: preparedResult.annotation,
+        modelAnnotation: preparedResult.modelAnnotation,
+        ...givenOptions,
+    };
+}
 /**
  * An object representing the task of loading a `PreparedResult`, capable of
  * materializing the prepared result (via `getPreparedResult()`) or extending the task run
@@ -2104,12 +2126,22 @@ class PreparedResultMaterializer extends FluentState {
     async run(options) {
         const preparedResult = await this.getPreparedResult();
         const connections = this.runtime.connections;
-        return Malloy.run({ connections, preparedResult, options });
+        const finalOptions = runSQLOptionsWithAnnotations(preparedResult, options);
+        return Malloy.run({
+            connections,
+            preparedResult,
+            options: finalOptions,
+        });
     }
     async *runStream(options) {
         const preparedResult = await this.getPreparedResult();
         const connections = this.runtime.connections;
-        const stream = Malloy.runStream({ connections, preparedResult, options });
+        const finalOptions = runSQLOptionsWithAnnotations(preparedResult, options);
+        const stream = Malloy.runStream({
+            connections,
+            preparedResult,
+            options: finalOptions,
+        });
         for await (const row of stream) {
             yield row;
         }

package/dist/run_sql_options.d.ts

@@ -1,4 +1,7 @@
+import { Annotation } from './model/malloy_types';
 export interface RunSQLOptions {
     rowLimit?: number;
     abortSignal?: AbortSignal;
+    modelAnnotation?: Annotation;
+    queryAnnotation?: Annotation;
 }

package/dist/runtime_types.d.ts

@@ -1,5 +1,5 @@
 import { RunSQLOptions } from './run_sql_options';
-import { MalloyQueryData, QueryDataRow, QueryRunStats, SQLBlock, StructDef } from './model/malloy_types';
+import { Annotation, MalloyQueryData, QueryDataRow, QueryRunStats, SQLBlock, StructDef } from './model/malloy_types';
 import { Dialect } from './dialect';
 /**
  * The contents of a Malloy query document.
@@ -34,6 +34,7 @@ export interface URLReader {
  */
 export interface FetchSchemaOptions {
     refreshTimestamp?: number;
+    modelAnnotation?: Annotation;
 }
 /**
  * An object capable of reading schemas for given table names.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.104-dev231116200719",
+  "version": "0.0.104-dev231117214047",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",