@malloydata/malloy 0.0.294 → 0.0.296

package/dist/model/composite_source_utils.js

@@ -21,13 +21,13 @@ exports.fieldUsageJoinPaths = fieldUsageJoinPaths;
 exports.checkRequiredGroupBys = checkRequiredGroupBys;
 exports.pathEq = pathEq;
 exports.pathBegins = pathBegins;
-exports.sortFieldUsageByReferenceLocation = sortFieldUsageByReferenceLocation;
 exports.hasCompositesAnywhere = hasCompositesAnywhere;
 exports.logCompositeError = logCompositeError;
 exports.compileFilterExpression = compileFilterExpression;
 const malloy_filter_1 = require("@malloydata/malloy-filter");
 const malloy_types_1 = require("./malloy_types");
 const utils_1 = require("./utils");
+const utils_2 = require("../lang/utils");
 function _resolveCompositeSources(path, source, rootFields, nests, fieldUsage, 
 // for resolving nested composites; the list of sources to try
 sources) {
@@ -403,27 +403,40 @@ function resolveCompositeSources(source, segment, fieldUsage) {
 function fieldUsagePaths(fieldUsage) {
     return fieldUsage.map(u => u.path);
 }
-function formatFieldUsages(fieldUsage) {
+function dedupPaths(paths) {
     const deduped = [];
-    for (const usage of fieldUsage) {
-        if (!deduped.some(p => pathEq(p, usage.path))) {
-            deduped.push(usage.path);
+    for (const path of paths) {
+        if (!deduped.some(p => pathEq(p, path))) {
+            deduped.push(path);
         }
     }
+    return deduped;
+}
+function formatPaths(paths, combinator = 'and') {
+    const deduped = dedupPaths(paths);
     const formattedUsages = deduped.map(fieldUsage => formatFieldUsage(fieldUsage));
-    if (formattedUsages.length === 0) {
+    return commaAndList(formattedUsages, combinator);
+}
+function formatRequiredGroupings(requiredGroupings) {
+    return formatPaths(requiredGroupings.map(g => g.path), 'and/or');
+}
+function commaAndList(strs, combinator = 'and') {
+    if (strs.length === 0) {
         return '';
     }
-    else if (formattedUsages.length === 1) {
-        return formattedUsages[0];
+    else if (strs.length === 1) {
+        return strs[0];
     }
-    else if (formattedUsages.length === 2) {
-        return `${formattedUsages[0]} and ${formattedUsages[1]}`;
+    else if (strs.length === 2) {
+        return `${strs[0]} ${combinator} ${strs[1]}`;
     }
     else {
-        return `${formattedUsages.slice(0, -1).join(', ')}, and ${formattedUsages[formattedUsages.length - 1]}`;
+        return `${strs.slice(0, -1).join(', ')}, ${combinator} ${strs[strs.length - 1]}`;
     }
 }
+function formatFieldUsages(fieldUsage) {
+    return formatPaths(fieldUsage.map(u => u.path));
+}
 function countFieldUsage(fieldUsage) {
     const paths = [];
     for (const usage of fieldUsage) {
@@ -529,6 +542,7 @@ function requiredGroupBysAt(requiredGroupBys, at) {
         return requiredGroupBys;
     return requiredGroupBys === null || requiredGroupBys === void 0 ? void 0 : requiredGroupBys.map(r => ({
         ...r,
+        fieldUsage: r.fieldUsage ? fieldUsageAt([r.fieldUsage], at)[0] : undefined,
         at,
     }));
 }
@@ -698,7 +712,7 @@ function expandRefs(nests, fields) {
                         missingFields.push(field);
                         continue;
                     }
-                    requiredGroupBys.push({ path, at: field.at });
+                    requiredGroupBys.push({ path, at: field.at, fieldUsage: field });
                 }
             }
             if (def.ungroupings) {
@@ -827,8 +841,21 @@ function compareLocations(a, b) {
         return 1;
     return 0;
 }
-function sortFieldUsageByReferenceLocation(usage) {
-    return usage.sort((a, b) => compareLocations(a.at, b.at));
+function issueLocation(issue) {
+    if (issue.type === 'missing-field') {
+        return issue.field.at;
+    }
+    else if (issue.type === 'missing-required-group-by') {
+        return issue.requiredGroupBy.at;
+    }
+    else {
+        return issue.firstUsage.at;
+    }
+}
+function sortIssuesByReferenceLocation(issues) {
+    return issues.sort((a, b) => {
+        return compareLocations(issueLocation(a), issueLocation(b));
+    });
 }
 function hasCompositesAnywhere(source) {
     if (source.type === 'composite')
@@ -840,55 +867,60 @@ function hasCompositesAnywhere(source) {
     }
     return false;
 }
+function issueFieldUsage(issue) {
+    if (issue.type === 'missing-field') {
+        return issue.field;
+    }
+    else if (issue.type === 'missing-required-group-by') {
+        return issue.requiredGroupBy.fieldUsage;
+    }
+    else {
+        return undefined;
+    }
+}
 function logCompositeError(error, logTo) {
     if (error.code === 'no_suitable_composite_source_input') {
-        if (error.data.failures.length > 0 &&
-            error.data.failures.every(failure => {
-                return (failure.issues.length > 0 &&
-                    failure.issues.every(issue => issue.type === 'missing-field'));
-            })) {
-            const firstFails = error.data.failures.map(failure => {
-                if (failure.issues[0].type !== 'missing-field')
-                    throw new Error('Programmer error');
-                return failure.issues[0].field;
-            });
-            const sorted = sortFieldUsageByReferenceLocation(firstFails);
-            const lastUsage = sorted[sorted.length - 1];
-            logTo.logError('invalid-composite-field-usage', {
-                newUsage: [lastUsage],
-                conflictingUsage: sorted,
-                allUsage: error.data.usage,
-            }, {
-                at: lastUsage.at,
-            });
-        }
-        else {
-            for (let i = 0; i < error.data.failures.length; i++) {
-                const failure = error.data.failures[i];
-                const sourceName = failure.source.as
-                    ? ` (\`${failure.source.as}\`)`
-                    : '';
-                const join = error.data.path.length === 0
-                    ? ''
-                    : `join ${error.data.path.join('.')} of `;
-                const source = `${join}composed source #${i + 1}${sourceName}`;
-                const requiredFields = `\nFields required in source: ${formatFieldUsages(error.data.usage)}`;
-                for (const issue of failure.issues) {
-                    if (issue.type === 'missing-field') {
-                        const fieldRef = `\`${issue.field.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: missing field ${fieldRef} in ${source}${requiredFields}`, { at: issue.field.at });
-                    }
-                    else if (issue.type === 'missing-required-group-by') {
-                        const fieldRef = `\`${issue.requiredGroupBy.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: missing group by or single value filter of ${fieldRef} as required in ${source}${requiredFields}`, { at: issue.requiredGroupBy.at });
-                    }
-                    else {
-                        const joinRef = `\`${issue.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: join ${joinRef} could not be resolved in ${source}${requiredFields}`, { at: issue.firstUsage.at });
-                    }
-                }
-            }
-        }
+        const firstFails = error.data.failures.map(failure => failure.issues[0]);
+        const sorted = sortIssuesByReferenceLocation(firstFails);
+        const usages = sorted.map(issueFieldUsage);
+        const lastIssue = sorted[sorted.length - 1];
+        const lastUsage = usages[usages.length - 1];
+        const conflictingUsage = firstFails
+            .filter(i => i.type === 'missing-field')
+            .map(i => i.field);
+        const fConflictingUsage = formatFieldUsages(conflictingUsage);
+        const dConflictingUsage = conflictingUsage.length > 0
+            ? `there is no composite input source which defines all of ${fConflictingUsage}`
+            : undefined;
+        const missingGroupBys = firstFails
+            .filter(i => i.type === 'missing-required-group-by')
+            .map(i => i.requiredGroupBy);
+        const fMissingGroupBys = formatRequiredGroupings(missingGroupBys);
+        const dGrouping = 'required group by or single value filter';
+        const dMissingGroupBys = missingGroupBys.length > 0
+            ? `there is a missing ${dGrouping} of ${fMissingGroupBys}`
+            : undefined;
+        const dConflictingUsageAndMissingGroupBys = conflictingUsage.length > 0 && missingGroupBys.length > 0
+            ? `there is no composite input source which defines ${fConflictingUsage} without having an unsatisfied ${dGrouping} on ${fMissingGroupBys}`
+            : undefined;
+        const failedJoins = firstFails
+            .filter(i => i.type === 'join-failed')
+            .map(i => i.path);
+        const uniqueFailedJoins = dedupPaths(failedJoins);
+        const joinPlural = uniqueFailedJoins.length > 1 ? 'joins' : 'join';
+        const dFailedJoins = failedJoins.length > 0
+            ? `${joinPlural} ${formatPaths(uniqueFailedJoins)} could not be resolved`
+            : undefined;
+        const dLastIssue = lastUsage
+            ? `uses field ${formatFieldUsages([lastUsage])}, resulting in`
+            : 'results in';
+        const dIssues = dConflictingUsageAndMissingGroupBys
+            ? commaAndList([dConflictingUsageAndMissingGroupBys, dFailedJoins].filter(utils_2.isNotUndefined))
+            : commaAndList([dConflictingUsage, dMissingGroupBys, dFailedJoins].filter(utils_2.isNotUndefined));
+        const message = `This operation ${dLastIssue} invalid usage of the composite source, as ${dIssues} (fields required in source: ${formatFieldUsages(error.data.usage)})`;
+        logTo.logError('could-not-resolve-composite-source', message, {
+            at: issueLocation(lastIssue),
+        });
     }
     else {
         logTo.logError('could-not-resolve-composite-source', 'Could not resolve composite source');

package/dist/model/malloy_query.d.ts

@@ -1,3 +1,4 @@
+import type * as Malloy from '@malloydata/malloy-interfaces';
 import type { QueryInfo, Dialect, DialectFieldList } from '../dialect';
 import type { AggregateFunctionType, CompiledQuery, Expr, FieldDef, Filtered, FunctionOverloadDef, FunctionParameterDef, JoinRelationship, ModelDef, OrderBy, OutputFieldNode, ParameterNode, PipeSegment, Query, QueryFieldDef, QuerySegment, RefToField, ResultMetadataDef, ResultStructMetadataDef, SearchIndexResult, SegmentFieldDef, StructDef, StructRef, TurtleDef, FunctionOrderBy, Argument, AggregateExpr, FilterCondition, GenericSQLExpr, FieldnameNode, FunctionCallNode, UngroupNode, SourceReferenceNode, SpreadExpr, FilteredExpr, SourceDef, BooleanFieldDef, QueryResultDef, QueryToMaterialize, PrepareResultOptions, CaseExpr, BasicAtomicDef, AtomicFieldDef, FilterMatchExpr } from './malloy_types';
 import type { Connection } from '../connection/types';
@@ -126,11 +127,12 @@ declare class FieldInstanceField implements FieldInstance {
     f: QueryField;
     fieldUsage: FieldUsage;
     parent: FieldInstanceResult;
+    readonly drillExpression: Malloy.Expression | undefined;
     type: FieldInstanceType;
     additionalGroupSets: number[];
     analyticalSQL: string | undefined;
     partitionSQL: string | undefined;
-    constructor(f: QueryField, fieldUsage: FieldUsage, parent: FieldInstanceResult);
+    constructor(f: QueryField, fieldUsage: FieldUsage, parent: FieldInstanceResult, drillExpression: Malloy.Expression | undefined);
     root(): FieldInstanceResultRoot;
     getSQL(): string;
     getAnalyticalSQL(forPartition: boolean): string;
@@ -161,7 +163,7 @@ declare class FieldInstanceResult implements FieldInstance {
      * @returns QueryInfo
      */
     getQueryInfo(): QueryInfo;
-    addField(as: string, field: QueryField, usage: FieldUsage): void;
+    addField(as: string, field: QueryField, usage: FieldUsage, drillExpression: Malloy.Expression | undefined): void;
     parentGroupSet(): number;
     add(name: string, f: FieldInstance): void;
     hasField(name: string): boolean;
@@ -259,6 +261,7 @@ declare class QueryQuery extends QueryField {
     addDependantExpr(resultStruct: FieldInstanceResult, context: QueryStruct, e: Expr, joinStack: string[]): void;
     addDependancies(resultStruct: FieldInstanceResult, field: QueryField): void;
     getSegmentFields(resultStruct: FieldInstanceResult): SegmentFieldDef[];
+    private getDrillExpression;
     expandFields(resultStruct: FieldInstanceResult): void;
     expandFilters(resultStruct: FieldInstanceResult): void;
     generateSQLFilters(resultStruct: FieldInstanceResult, which: 'where' | 'having'): AndChain;

package/dist/model/malloy_query.js

@@ -1,8 +1,4 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.QueryModel = exports.Segment = void 0;
-exports.getResultStructDefForView = getResultStructDefForView;
-exports.getResultStructDefForQuery = getResultStructDefForQuery;
 /*
  * Copyright 2023 Google LLC
  *
@@ -25,6 +21,10 @@ exports.getResultStructDefForQuery = getResultStructDefForQuery;
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueryModel = exports.Segment = void 0;
+exports.getResultStructDefForView = getResultStructDefForView;
+exports.getResultStructDefForQuery = getResultStructDefForQuery;
 const uuid_1 = require("uuid");
 const dialect_1 = require("../dialect");
 const standardsql_1 = require("../dialect/standardsql/standardsql");
@@ -1124,10 +1124,11 @@ function sqlSumDistinct(dialect, sqlExp, sqlDistintKey) {
     return ret;
 }
 class FieldInstanceField {
-    constructor(f, fieldUsage, parent) {
+    constructor(f, fieldUsage, parent, drillExpression) {
         this.f = f;
         this.fieldUsage = fieldUsage;
         this.parent = parent;
+        this.drillExpression = drillExpression;
         this.type = 'field';
         this.additionalGroupSets = [];
     }
@@ -1186,7 +1187,7 @@ class FieldInstanceResult {
         }
         return {};
     }
-    addField(as, field, usage) {
+    addField(as, field, usage, drillExpression) {
         const fi = this.allFields.get(as);
         if (fi) {
             if (fi.type === 'query') {
@@ -1203,7 +1204,7 @@ class FieldInstanceResult {
                 }
             }
         }
-        this.add(as, new FieldInstanceField(field, usage, this));
+        this.add(as, new FieldInstanceField(field, usage, this, drillExpression));
     }
     parentGroupSet() {
         if (this.parent) {
@@ -1828,10 +1829,16 @@ class QueryQuery extends QueryField {
                 ? fs.queryFields
                 : [];
     }
+    getDrillExpression(f) {
+        if ((0, malloy_types_1.isAtomic)(f) || f.type === 'fieldref')
+            return f.drillExpression;
+        return undefined;
+    }
     expandFields(resultStruct) {
         let resultIndex = 1;
         for (const f of this.getSegmentFields(resultStruct)) {
             const { as, field } = this.expandField(f);
+            const drillExpression = this.getDrillExpression(f);
             if (field instanceof QueryQuery) {
                 if (this.firstSegment.type === 'project') {
                     throw new Error(`Nested views cannot be used in select - '${field.fieldDef.name}'`);
@@ -1844,7 +1851,7 @@ class QueryQuery extends QueryField {
                 resultStruct.addField(as, field, {
                     resultIndex,
                     type: 'result',
-                });
+                }, drillExpression);
                 this.addDependancies(resultStruct, field);
                 if (isBasicAggregate(field)) {
                     if (this.firstSegment.type === 'project') {
@@ -1859,7 +1866,7 @@ class QueryQuery extends QueryField {
                 resultStruct.addField(as, field, {
                     resultIndex,
                     type: 'result',
-                });
+                }, drillExpression);
             }
             // else if (
             //   this.firstSegment.type === "project" &&
@@ -1950,11 +1957,13 @@ class QueryQuery extends QueryField {
                     : undefined;
                 const sourceClasses = [sourceField];
                 const referenceId = fi.f.referenceId;
+                const drillExpression = fi.drillExpression;
                 const base = {
                     sourceField,
                     sourceExpression,
                     sourceClasses,
                     referenceId,
+                    drillExpression,
                 };
                 if (isBasicCalculation(fi.f)) {
                     filterList = fi.f.getFilterList();
@@ -2020,7 +2029,6 @@ class QueryQuery extends QueryField {
                         join: 'many',
                         name,
                         resultMetadata,
-                        drillView: fi.turtleDef.drillView,
                     };
                     fields.push(multiLineNest);
                 }
@@ -2031,7 +2039,6 @@ class QueryQuery extends QueryField {
                         join: 'one',
                         name,
                         resultMetadata,
-                        drillView: fi.turtleDef.drillView,
                     };
                     fields.push(oneLineNest);
                 }
@@ -2060,12 +2067,10 @@ class QueryQuery extends QueryField {
                     }
                     const location = fOut.location;
                     const annotation = fOut.annotation;
-                    const drillView = (0, malloy_types_1.isAtomic)(fOut) || (0, malloy_types_1.isTurtle)(fOut) ? fOut.drillView : undefined;
                     const common = {
                         resultMetadata,
                         location,
                         annotation,
-                        drillView,
                     };
                     // build out the result fields...
                     switch (fOut.type) {
@@ -2835,7 +2840,6 @@ class QueryQuery extends QueryField {
                 type: 'turtle',
                 name: 'starthere',
                 pipeline,
-                drillView: fi.turtleDef.drillView,
             };
             const inputStruct = {
                 type: 'nest_source',
@@ -2942,11 +2946,12 @@ class QueryQueryIndexStage extends QueryQuery {
         this.maxGroupSet = groupIndex;
         for (const f of this.firstSegment.indexFields) {
             const { as, field } = this.expandField(f);
-            this.indexPaths[as] = f.path;
+            const referencePath = f.path;
+            this.indexPaths[as] = referencePath;
             resultStruct.addField(as, field, {
                 resultIndex,
                 type: 'result',
-            });
+            }, undefined);
             if (field instanceof QueryAtomicField) {
                 this.addDependancies(resultStruct, field);
             }
@@ -2958,7 +2963,7 @@ class QueryQueryIndexStage extends QueryQuery {
             resultStruct.addField(measure, f, {
                 resultIndex,
                 type: 'result',
-            });
+            }, undefined);
             this.addDependancies(resultStruct, f);
         }
         this.expandFilters(resultStruct);
@@ -3615,20 +3620,20 @@ class QueryStruct {
         return field;
     }
     getQueryFieldReference(f) {
-        const { path, annotation, drillView } = f;
+        const { path, annotation, drillExpression } = f;
         const field = this.getFieldByName(path);
-        if (annotation || drillView) {
+        if (annotation || drillExpression) {
             if (field.parent === undefined) {
                 throw new Error('Inconcievable, field reference to orphaned query field');
             }
             // Made a field object from the source, but the annotations were computed by the compiler
             // when it generated the reference, and has both the source and reference annotations included.
             if (field instanceof QueryFieldStruct) {
-                const newDef = { ...field.fieldDef, annotation, drillView };
+                const newDef = { ...field.fieldDef, annotation, drillExpression };
                 return new QueryFieldStruct(newDef, undefined, field.parent, {}, field.referenceId);
             }
             else {
-                const newDef = { ...field.fieldDef, annotation, drillView };
+                const newDef = { ...field.fieldDef, annotation, drillExpression };
                 return field.parent.makeQueryField(newDef, field.referenceId);
             }
         }
@@ -3683,7 +3688,6 @@ class QueryStruct {
             pipeline,
             annotation,
             location: turtleDef.location,
-            drillView: turtleDef.drillView,
         };
         return flatTurtleDef;
     }

package/dist/model/malloy_types.d.ts

@@ -1,4 +1,4 @@
-import type { Filter } from '@malloydata/malloy-interfaces';
+import type * as Malloy from '@malloydata/malloy-interfaces';
 /**
  * Field computations are compiled into an expression tree of "Expr"
  * type nodes. Each node is one of these three interfaces. The
@@ -60,8 +60,8 @@ export interface FilterCondition extends ExprE {
     code: string;
     expressionType: ExpressionType;
     fieldUsage?: FieldUsage[];
-    drillView?: string;
-    stableFilter?: Filter;
+    filterView?: string;
+    stableFilter?: Malloy.Filter;
     isSourceFilter?: boolean;
 }
 export interface FilteredExpr extends ExprWithKids {
@@ -265,6 +265,7 @@ export interface Expression {
     fieldUsage?: FieldUsage[];
     expressionType?: ExpressionType;
     code?: string;
+    drillExpression?: Malloy.Expression;
 }
 type ConstantExpr = Expr;
 interface ParameterInfo {
@@ -343,6 +344,7 @@ export interface ResultMetadataDef {
     filterList?: FilterCondition[];
     fieldKind: 'measure' | 'dimension' | 'struct';
     referenceId?: string;
+    drillExpression?: Malloy.Expression | undefined;
     drillable?: boolean;
 }
 export interface Ordered {
@@ -389,7 +391,7 @@ export interface FieldBase extends NamedObject, Expression, ResultMetadata {
     accessModifier?: NonDefaultAccessModifierLabel | undefined;
     requiresGroupBy?: RequiredGroupBy[];
     ungroupings?: AggregateUngrouping[];
-    drillView?: string;
+    drillExpression?: Malloy.Expression | undefined;
 }
 export declare function fieldIsIntrinsic(f: FieldDef): f is AtomicFieldDef;
 export interface StringTypeDef {
@@ -610,7 +612,6 @@ export interface TurtleDef extends NamedObject, Pipeline {
     accessModifier?: NonDefaultAccessModifierLabel | undefined;
     fieldUsage?: FieldUsage[];
     requiredGroupBys?: string[][];
-    drillView?: string;
 }
 interface StructDefBase extends HasLocation, NamedObject {
     type: string;
@@ -684,6 +685,7 @@ export type ExpressionValueType = AtomicFieldType | NonAtomicType;
 export type ExpressionValueTypeDef = AtomicTypeDef | NonAtomicTypeDef;
 export type BasicExpressionType = Exclude<ExpressionValueType, JoinElementType | 'turtle'>;
 export interface RequiredGroupBy {
+    fieldUsage?: FieldUsage;
     at?: DocumentLocation;
     path: string[];
 }
@@ -799,7 +801,7 @@ export interface RefToField {
     path: string[];
     annotation?: Annotation;
     at?: DocumentLocation;
-    drillView?: string;
+    drillExpression?: Malloy.Expression | undefined;
 }
 export type QueryFieldDef = AtomicFieldDef | TurtleDef | RefToField;
 export type TypedDef = AtomicTypeDef | JoinFieldDef | TurtleDef | RefToField | StructDef;

package/dist/to_stable.d.ts

@@ -1,4 +1,4 @@
-import type * as Malloy from '@malloydata/malloy-interfaces';
+import * as Malloy from '@malloydata/malloy-interfaces';
 import type { FieldDef, ModelDef, ResultStructMetadataDef, SourceDef } from './model';
 import { Tag } from '@malloydata/malloy-tag';
 export declare function sourceDefToSourceInfo(sourceDef: SourceDef): Malloy.SourceInfo;
@@ -6,3 +6,15 @@ export declare function modelDefToModelInfo(modelDef: ModelDef): Malloy.ModelInf
 export declare function convertFieldInfos(source: SourceDef, fields: FieldDef[]): Malloy.FieldInfo[];
 export declare function writeLiteralToTag(tag: Tag, path: (string | number)[], literal: Malloy.LiteralValue): void;
 export declare function getResultStructMetadataAnnotation(field: SourceDef, resultMetadata: ResultStructMetadataDef): Malloy.Annotation | undefined;
+/**
+ * Writes a Malloy interface object to a tag at a given path.
+ *
+ * E.g. `writeMalloyObjectToTag(tag, ['expr'], 'Expression', {kind: 'field_reference', name: 'carrier'})`
+ *
+ * produces the tag `#(malloy) expr { kind = field_reference name = carrier }`
+ */
+export declare function writeMalloyObjectToTag(tag: Tag, path: (string | number)[], obj: unknown, type: string): void;
+/**
+ * Extracts a Malloy interface object from a tag at a given path; the inverse of `writeMalloyObjectToTag`.
+ */
+export declare function extractMalloyObjectFromTag(tag: Tag, type: string): unknown;