@malloydata/malloy 0.0.221-dev241209191943 → 0.0.221-dev241211195814

package/dist/lang/ast/field-space/index-field-space.js

@@ -23,6 +23,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IndexFieldSpace = void 0;
+const composite_source_utils_1 = require("../../../model/composite_source_utils");
 const malloy_types_1 = require("../../../model/malloy_types");
 const field_references_1 = require("../query-items/field-references");
 const space_field_1 = require("../types/space-field");
@@ -52,15 +53,20 @@ class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
             this.logError('refinement-of-index-segment', 'index query operations cannot be refined');
             return { type: 'index', indexFields: [] };
         }
+        let compositeFieldUsage = (0, composite_source_utils_1.emptyCompositeFieldUsage)();
+        let narrowedCompositeFieldResolution = (0, composite_source_utils_1.emptyNarrowedCompositeFieldResolution)();
         const indexFields = [];
+        const source = this.inputSpace().structDef();
         for (const [name, field] of this.entries()) {
             if (field instanceof space_field_1.SpaceField) {
-                const wildPath = this.expandedWild[name];
-                if (wildPath) {
-                    indexFields.push({ type: 'fieldref', path: wildPath });
-                    continue;
+                let nextCompositeFieldUsage = undefined;
+                let logTo = undefined;
+                const wild = this.expandedWild[name];
+                if (wild) {
+                    indexFields.push({ type: 'fieldref', path: wild.path });
+                    nextCompositeFieldUsage = wild.entry.typeDesc().compositeFieldUsage;
                 }
-                if (field instanceof reference_field_1.ReferenceField) {
+                else if (field instanceof reference_field_1.ReferenceField) {
                     // attempt to cause a type check
                     const fieldRef = field.fieldRef;
                     const check = fieldRef.getField(this.exprSpace);
@@ -69,10 +75,18 @@ class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
                     }
                     else {
                         indexFields.push(fieldRef.refToField);
+                        nextCompositeFieldUsage =
+                            check.found.typeDesc().compositeFieldUsage;
+                        logTo = fieldRef;
                     }
                 }
+                const next = this.applyNextCompositeFieldUsage(source, compositeFieldUsage, narrowedCompositeFieldResolution, nextCompositeFieldUsage, logTo);
+                compositeFieldUsage = next.compositeFieldUsage;
+                narrowedCompositeFieldResolution =
+                    next.narrowedCompositeFieldResolution;
             }
         }
+        this._compositeFieldUsage = compositeFieldUsage;
         return { type: 'index', indexFields };
     }
     addRefineFromFields(_refineThis) { }
@@ -115,7 +129,7 @@ class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
                 name,
             ]);
             if (this.entry(indexName)) {
-                const conflict = (_a = this.expandedWild[indexName]) === null || _a === void 0 ? void 0 : _a.join('.');
+                const conflict = (_a = this.expandedWild[indexName].path) === null || _a === void 0 ? void 0 : _a.join('.');
                 wild.logError('name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
             }
             else {
@@ -125,7 +139,10 @@ class IndexFieldSpace extends query_spaces_1.QueryOperationSpace {
                     (0, malloy_types_1.expressionIsScalar)(eTypeDesc.expressionType) &&
                     (dialect === undefined || !dialect.ignoreInProject(name))) {
                     expandEntries.push({ name: indexName, entry });
-                    this.expandedWild[indexName] = joinPath.concat(name);
+                    this.expandedWild[indexName] = {
+                        path: joinPath.concat(name),
+                        entry,
+                    };
                 }
             }
         }

package/dist/lang/ast/field-space/query-spaces.d.ts

@@ -8,6 +8,7 @@ import { LookupResult } from '../types/lookup-result';
 import { QueryInputSpace } from './query-input-space';
 import { SpaceEntry } from '../types/space-entry';
 import { LogMessageOptions, MessageCode, MessageParameterType } from '../../parse-log';
+import { NarrowedCompositeFieldResolution } from '../../../model/composite_source_utils';
 /**
  * The output space of a query operation. It is not named "QueryOutputSpace"
  * because this is the namespace of the Query which is a layer of an output and
@@ -20,7 +21,10 @@ export declare abstract class QueryOperationSpace extends RefinedSpace implement
     readonly astEl: MalloyElement;
     protected exprSpace: QueryInputSpace;
     abstract readonly segmentType: 'reduce' | 'project' | 'index';
-    expandedWild: Record<string, string[]>;
+    expandedWild: Record<string, {
+        path: string[];
+        entry: SpaceEntry;
+    }>;
     compositeFieldUsers: ({
         type: 'filter';
         filter: model.FilterCondition;
@@ -44,6 +48,10 @@ export declare abstract class QueryOperationSpace extends RefinedSpace implement
     protected getCompositeFieldUsageIncludingJoinOns(compositeFieldUsage: model.CompositeFieldUsage): model.CompositeFieldUsage;
     addCompositeFieldUserFromFilter(filter: model.FilterCondition, logTo: MalloyElement): void;
     newEntry(name: string, logTo: MalloyElement, entry: SpaceEntry): void;
+    protected applyNextCompositeFieldUsage(source: model.SourceDef, compositeFieldUsage: model.CompositeFieldUsage, narrowedCompositeFieldResolution: NarrowedCompositeFieldResolution, nextCompositeFieldUsage: model.CompositeFieldUsage | undefined, logTo: MalloyElement | undefined): {
+        compositeFieldUsage: model.CompositeFieldUsage;
+        narrowedCompositeFieldResolution: NarrowedCompositeFieldResolution;
+    };
 }
 export declare abstract class QuerySpace extends QueryOperationSpace {
     addRefineFromFields(refineThis: model.PipeSegment): void;

package/dist/lang/ast/field-space/query-spaces.js

@@ -77,7 +77,8 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
         this.astEl = astEl;
         this.expandedWild = {};
         this.compositeFieldUsers = [];
-        // Composite field usage is not computed until `queryFieldDefs` is called; if anyone
+        // Composite field usage is not computed until `queryFieldDefs` is called
+        // (or `getPipeSegment` for index segments); if anyone
         // tries to access it before that, they'll get an error
         this._compositeFieldUsage = undefined;
         this.exprSpace = new query_input_space_1.QueryInputSpace(queryInputSpace.structDef(), this);
@@ -131,7 +132,7 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
                 continue;
             }
             if (this.entry(name)) {
-                const conflict = (_a = this.expandedWild[name]) === null || _a === void 0 ? void 0 : _a.join('.');
+                const conflict = (_a = this.expandedWild[name]) === null || _a === void 0 ? void 0 : _a.path.join('.');
                 wild.logError('name-conflict-in-wildcard-expansion', `Cannot expand '${name}' in '${wild.refString}' because a field with that name already exists${conflict ? ` (conflicts with ${conflict})` : ''}`);
             }
             else {
@@ -140,7 +141,10 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
                     model.expressionIsScalar(eType.expressionType) &&
                     (dialect === undefined || !dialect.ignoreInProject(name))) {
                     expandEntries.push({ name, entry });
-                    this.expandedWild[name] = joinPath.concat(name);
+                    this.expandedWild[name] = {
+                        path: joinPath.concat(name),
+                        entry,
+                    };
                 }
             }
         }
@@ -163,7 +167,7 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
         var _a;
         const reference = joinPath.map(n => new field_space_1.FieldName(n));
         this.astEl.has({ reference });
-        const lookup = this.queryInputSpace.lookup(reference);
+        const lookup = this.exprSpace.lookup(reference);
         // Should always be found...
         if (lookup.found && lookup.found instanceof struct_space_field_base_1.StructSpaceFieldBase) {
             return ((_a = lookup.found.fieldDef().onCompositeFieldUsage) !== null && _a !== void 0 ? _a : (0, composite_source_utils_1.emptyCompositeFieldUsage)());
@@ -189,6 +193,26 @@ class QueryOperationSpace extends refined_space_1.RefinedSpace {
         }
         super.newEntry(name, logTo, entry);
     }
+    applyNextCompositeFieldUsage(source, compositeFieldUsage, narrowedCompositeFieldResolution, nextCompositeFieldUsage, logTo) {
+        if (nextCompositeFieldUsage) {
+            const newCompositeFieldUsage = this.getCompositeFieldUsageIncludingJoinOns((0, composite_source_utils_1.compositeFieldUsageDifference)(nextCompositeFieldUsage, compositeFieldUsage));
+            compositeFieldUsage = (0, composite_source_utils_1.mergeCompositeFieldUsage)(compositeFieldUsage, newCompositeFieldUsage);
+            if (!(0, composite_source_utils_1.isEmptyCompositeFieldUsage)(newCompositeFieldUsage)) {
+                const result = (0, composite_source_utils_1.narrowCompositeFieldResolution)(source, compositeFieldUsage, narrowedCompositeFieldResolution);
+                if (result.error) {
+                    (logTo !== null && logTo !== void 0 ? logTo : this).logError('invalid-composite-field-usage', {
+                        newUsage: newCompositeFieldUsage,
+                        allUsage: compositeFieldUsage,
+                    });
+                }
+                else {
+                    narrowedCompositeFieldResolution =
+                        result.narrowedCompositeFieldResolution;
+                }
+            }
+        }
+        return { compositeFieldUsage, narrowedCompositeFieldResolution };
+    }
 }
 exports.QueryOperationSpace = QueryOperationSpace;
 // Project and Reduce or "QuerySegments" are built from a QuerySpace
@@ -252,57 +276,38 @@ class QuerySpace extends QueryOperationSpace {
                 const { name, field } = user;
                 const wildPath = this.expandedWild[name];
                 if (wildPath) {
-                    fields.push({ type: 'fieldref', path: wildPath });
-                    continue;
+                    fields.push({ type: 'fieldref', path: wildPath.path });
+                    nextCompositeFieldUsage =
+                        wildPath.entry.typeDesc().compositeFieldUsage;
                 }
-                // TODO handle wildcards for composite sources
-                const fieldQueryDef = field.getQueryFieldDef(this.exprSpace);
-                if (fieldQueryDef) {
-                    const typeDesc = field.typeDesc();
-                    nextCompositeFieldUsage = typeDesc.compositeFieldUsage;
-                    // Filter out fields whose type is 'error', which means that a totally bad field
-                    // isn't sent to the compiler, where it will wig out.
-                    // TODO Figure out how to make errors generated by `canContain` go in the right place,
-                    // maybe by adding a logable element to SpaceFields.
-                    if (typeDesc &&
-                        typeDesc.type !== 'error' &&
-                        this.canContain(typeDesc) &&
-                        !isEmptyNest(fieldQueryDef)) {
-                        fields.push(fieldQueryDef);
-                    }
-                }
-                // TODO I removed the error here because during calculation of the refinement space,
-                // (see creation of a QuerySpace) we add references to all the fields from
-                // the refinement, but they don't have definitions. So in the case where we
-                // don't have a field def, we "know" that that field is already in the query,
-                // and we don't need to worry about actually adding it. Previously, this was also true for
-                // project statements, where we added "*" as a field and also all the individual
-                // fields, but the individual fields didn't have field defs.
-            }
-            if (nextCompositeFieldUsage) {
-                const newCompositeFieldUsage = this.getCompositeFieldUsageIncludingJoinOns((0, composite_source_utils_1.compositeFieldUsageDifference)(nextCompositeFieldUsage, compositeFieldUsage));
-                compositeFieldUsage = (0, composite_source_utils_1.mergeCompositeFieldUsage)(compositeFieldUsage, newCompositeFieldUsage);
-                if (!(0, composite_source_utils_1.isEmptyCompositeFieldUsage)(newCompositeFieldUsage)) {
-                    const result = (0, composite_source_utils_1.narrowCompositeFieldResolution)(source, compositeFieldUsage, narrowedCompositeFieldResolution);
-                    if (result.error) {
-                        if (user.logTo) {
-                            user.logTo.logError('invalid-composite-field-usage', {
-                                newUsage: newCompositeFieldUsage,
-                                allUsage: compositeFieldUsage,
-                            });
-                        }
-                        else {
-                            // This should not happen; logTo should only be not set for a field from a refinement,
-                            // which should never fail the composite resolution
-                            throw new Error('Unexpected invalid composite field resolution');
+                else {
+                    const fieldQueryDef = field.getQueryFieldDef(this.exprSpace);
+                    if (fieldQueryDef) {
+                        const typeDesc = field.typeDesc();
+                        nextCompositeFieldUsage = typeDesc.compositeFieldUsage;
+                        // Filter out fields whose type is 'error', which means that a totally bad field
+                        // isn't sent to the compiler, where it will wig out.
+                        // TODO Figure out how to make errors generated by `canContain` go in the right place,
+                        // maybe by adding a logable element to SpaceFields.
+                        if (typeDesc &&
+                            typeDesc.type !== 'error' &&
+                            this.canContain(typeDesc) &&
+                            !isEmptyNest(fieldQueryDef)) {
+                            fields.push(fieldQueryDef);
                         }
                     }
-                    else {
-                        narrowedCompositeFieldResolution =
-                            result.narrowedCompositeFieldResolution;
-                    }
+                    // TODO I removed the error here because during calculation of the refinement space,
+                    // (see creation of a QuerySpace) we add references to all the fields from
+                    // the refinement, but they don't have definitions. So in the case where we
+                    // don't have a field def, we "know" that that field is already in the query,
+                    // and we don't need to worry about actually adding it. Previously, this was also true for
+                    // project statements, where we added "*" as a field and also all the individual
+                    // fields, but the individual fields didn't have field defs.
                 }
             }
+            const next = this.applyNextCompositeFieldUsage(source, compositeFieldUsage, narrowedCompositeFieldResolution, nextCompositeFieldUsage, user.logTo);
+            compositeFieldUsage = next.compositeFieldUsage;
+            narrowedCompositeFieldResolution = next.narrowedCompositeFieldResolution;
         }
         this._compositeFieldUsage = compositeFieldUsage;
         return fields;

package/dist/lang/ast/query-builders/index-builder.d.ts

@@ -1,4 +1,4 @@
-import { FilterCondition, PipeSegment, Sampling } from '../../../model/malloy_types';
+import { CompositeFieldUsage, FilterCondition, PipeSegment, Sampling } from '../../../model/malloy_types';
 import { FieldName, SourceFieldSpace } from '../types/field-space';
 import { Limit } from '../query-properties/limit';
 import { IndexFieldSpace } from '../field-space/index-field-space';
@@ -18,5 +18,6 @@ export declare class IndexBuilder implements QueryBuilder {
     readonly type = "index";
     constructor(inputFS: SourceFieldSpace, refineThis: PipeSegment | undefined, isNestIn: QueryOperationSpace | undefined, astEl: MalloyElement);
     execute(qp: QueryProperty): void;
+    get compositeFieldUsage(): CompositeFieldUsage;
     finalize(from: PipeSegment | undefined): PipeSegment;
 }

package/dist/lang/ast/query-builders/index-builder.js

@@ -30,6 +30,7 @@ const indexing_1 = require("../query-properties/indexing");
 const limit_1 = require("../query-properties/limit");
 const sampling_1 = require("../query-properties/sampling");
 const index_field_space_1 = require("../field-space/index-field-space");
+const composite_source_utils_1 = require("../../../model/composite_source_utils");
 class IndexBuilder {
     constructor(inputFS, refineThis, isNestIn, astEl) {
         this.filters = [];
@@ -64,7 +65,11 @@ class IndexBuilder {
             qp.logError('illegal-operation-for-index', 'Not legal in an index query operation');
         }
     }
+    get compositeFieldUsage() {
+        return this.resultFS.compositeFieldUsage;
+    }
     finalize(from) {
+        var _a;
         if (from && !(0, malloy_types_1.isIndexSegment)(from) && !(0, malloy_types_1.isPartialSegment)(from)) {
             this.resultFS.logError('refinement-of-index-segment', `Can't refine index with ${from.type}`);
             return error_factory_1.ErrorFactory.indexSegment;
@@ -95,6 +100,10 @@ class IndexBuilder {
         if (this.alwaysJoins.length > 0) {
             indexSegment.alwaysJoins = [...this.alwaysJoins];
         }
+        const fromCompositeFieldUsage = from && from.type === 'index'
+            ? (_a = from.compositeFieldUsage) !== null && _a !== void 0 ? _a : (0, composite_source_utils_1.emptyCompositeFieldUsage)()
+            : (0, composite_source_utils_1.emptyCompositeFieldUsage)();
+        indexSegment.compositeFieldUsage = (0, composite_source_utils_1.mergeCompositeFieldUsage)(fromCompositeFieldUsage, this.compositeFieldUsage);
         return indexSegment;
     }
 }

package/dist/lang/ast/query-elements/query-base.js

@@ -31,9 +31,10 @@ class QueryBase extends malloy_element_1.MalloyElement {
     query() {
         var _a;
         const { inputStruct, query } = this.queryComp(true);
-        // TODO add an error if a raw/index query is done against a composite source
+        // TODO add an error if a raw query is done against a composite source
         let compositeResolvedSourceDef = undefined;
-        if (query.pipeline[0] && (0, malloy_types_1.isQuerySegment)(query.pipeline[0])) {
+        if (query.pipeline[0] &&
+            ((0, malloy_types_1.isQuerySegment)(query.pipeline[0]) || (0, malloy_types_1.isIndexSegment)(query.pipeline[0]))) {
             const compositeFieldUsage = (_a = query.pipeline[0].compositeFieldUsage) !== null && _a !== void 0 ? _a : (0, composite_source_utils_1.emptyCompositeFieldUsage)();
             if (!(0, composite_source_utils_1.isEmptyCompositeFieldUsage)(compositeFieldUsage) ||
                 inputStruct.type === 'composite') {

package/dist/lang/test/composite-field-usage.spec.js

@@ -137,10 +137,15 @@ describe('composite sources', () => {
         ##! experimental.composite_sources
         run: compose(a, a extend { dimension: one is 1 }) -> { group_by: one }
     `).toTranslate());
-        test.skip('index on composite fails', () => expect(`
+        test('index on composite translates', () => expect(`
         ##! experimental.composite_sources
-        run: compose(a extend { dimension: two is 2 }, a extend { dimension: one is 1 }) -> { index: * }
-    `).toLog((0, test_translator_1.errorMessage)('Cannot index on a composite source')));
+        source: x is compose(
+          a extend { except: ai },
+          a
+        )
+        run: x -> { index: ai }
+        run: x -> { index: * }
+    `).toTranslate());
         test('raw run of composite source fails', () => expect(`
         ##! experimental.composite_sources
         run: compose(a extend { dimension: two is 2 }, a extend { dimension: one is 1 })

package/dist/model/composite_source_utils.d.ts

@@ -30,7 +30,7 @@ type SingleNarrowedCompositeFieldResolution = {
     source: SourceDef;
     nested?: SingleNarrowedCompositeFieldResolution | undefined;
 }[];
-interface NarrowedCompositeFieldResolution {
+export interface NarrowedCompositeFieldResolution {
     source: SingleNarrowedCompositeFieldResolution | undefined;
     joined: {
         [name: string]: NarrowedCompositeFieldResolution;

package/dist/model/composite_source_utils.js

@@ -10,11 +10,11 @@ exports.compositeFieldUsageJoinPaths = exports.joinedCompositeFieldUsage = expor
 const utils_1 = require("../lang/utils");
 const malloy_types_1 = require("./malloy_types");
 function _resolveCompositeSources(path, source, compositeFieldUsage, narrowedCompositeFieldResolution = undefined) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+    var _a, _b, _c, _d, _e, _f, _g;
     let base = { ...source };
     let narrowedSources = undefined;
     const nonCompositeFields = getNonCompositeFields(source);
-    if (compositeFieldUsage.fields.length > 0) {
+    if (compositeFieldUsage.fields.length > 0 || source.type === 'composite') {
         if (source.type === 'composite') {
             let found = false;
             // The narrowed source list is either the one given when this function was called,
@@ -86,18 +86,10 @@ function _resolveCompositeSources(path, source, compositeFieldUsage, narrowedCom
             return { error: { code: 'not_a_composite_source', data: { path } } };
         }
     }
-    else if (source.type === 'composite') {
-        const first = source.sources[0];
-        base = {
-            ...first,
-            fields: [...nonCompositeFields, ...base.fields],
-            filterList: [...((_e = source.filterList) !== null && _e !== void 0 ? _e : []), ...((_f = first.filterList) !== null && _f !== void 0 ? _f : [])],
-        };
-    }
     const fieldsByName = {};
-    const narrowedJoinedSources = (_g = narrowedCompositeFieldResolution === null || narrowedCompositeFieldResolution === void 0 ? void 0 : narrowedCompositeFieldResolution.joined) !== null && _g !== void 0 ? _g : {};
+    const narrowedJoinedSources = (_e = narrowedCompositeFieldResolution === null || narrowedCompositeFieldResolution === void 0 ? void 0 : narrowedCompositeFieldResolution.joined) !== null && _e !== void 0 ? _e : {};
     for (const field of base.fields) {
-        fieldsByName[(_h = field.as) !== null && _h !== void 0 ? _h : field.name] = field;
+        fieldsByName[(_f = field.as) !== null && _f !== void 0 ? _f : field.name] = field;
     }
     for (const [joinName, joinedUsage] of Object.entries(compositeFieldUsage.joinedUsage)) {
         const join = fieldsByName[joinName];
@@ -127,7 +119,7 @@ function _resolveCompositeSources(path, source, compositeFieldUsage, narrowedCom
         fieldsByName[joinName] = {
             ...resolved.success,
             join: join.join,
-            as: (_j = join.as) !== null && _j !== void 0 ? _j : join.name,
+            as: (_g = join.as) !== null && _g !== void 0 ? _g : join.name,
             onExpression: join.onExpression,
         };
         narrowedJoinedSources[joinName] = resolved.narrowedCompositeFieldResolution;
@@ -233,14 +225,12 @@ function arrayDifference(a, b) {
 function compositeFieldUsageDifference(a, b) {
     return {
         fields: arrayDifference(a.fields, b.fields),
-        joinedUsage: Object.fromEntries(Object.entries(a.joinedUsage)
-            .map(([joinName, joinedUsage]) => [
+        joinedUsage: Object.fromEntries(Object.entries(a.joinedUsage).map(([joinName, joinedUsage]) => [
             joinName,
             joinName in b.joinedUsage
                 ? compositeFieldUsageDifference(joinedUsage, b.joinedUsage[joinName])
                 : joinedUsage,
-        ])
-            .filter(([_, joinedUsage]) => countCompositeFieldUsage(joinedUsage) > 0)),
+        ])),
     };
 }
 exports.compositeFieldUsageDifference = compositeFieldUsageDifference;

package/dist/model/malloy_types.d.ts

@@ -546,6 +546,7 @@ export interface IndexSegment extends Filtered {
     weightMeasure?: string;
     sample?: Sampling;
     alwaysJoins?: string[];
+    compositeFieldUsage?: CompositeFieldUsage;
 }
 export declare function isIndexSegment(pe: PipeSegment): pe is IndexSegment;
 export interface CompositeFieldUsage {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.221-dev241209191943",
+  "version": "0.0.221-dev241211195814",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",