@malloydata/malloy 0.0.215 → 0.0.216-dev241118202522

package/dist/lang/lib/Malloy/MalloyParserListener.d.ts

@@ -78,6 +78,7 @@ import { JoinWithContext } from "./MalloyParser";
 import { JoinOnContext } from "./MalloyParser";
 import { SQIDContext } from "./MalloyParser";
 import { SQParensContext } from "./MalloyParser";
+import { SQComposeContext } from "./MalloyParser";
 import { SQRefinedQueryContext } from "./MalloyParser";
 import { SQArrowContext } from "./MalloyParser";
 import { SQExtendedSourceContext } from "./MalloyParser";
@@ -1177,6 +1178,18 @@ export interface MalloyParserListener extends ParseTreeListener {
      * @param ctx the parse tree
      */
     exitSQParens?: (ctx: SQParensContext) => void;
+    /**
+     * Enter a parse tree produced by the `SQCompose`
+     * labeled alternative in `MalloyParser.sqExpr`.
+     * @param ctx the parse tree
+     */
+    enterSQCompose?: (ctx: SQComposeContext) => void;
+    /**
+     * Exit a parse tree produced by the `SQCompose`
+     * labeled alternative in `MalloyParser.sqExpr`.
+     * @param ctx the parse tree
+     */
+    exitSQCompose?: (ctx: SQComposeContext) => void;
     /**
      * Enter a parse tree produced by the `SQRefinedQuery`
      * labeled alternative in `MalloyParser.sqExpr`.

package/dist/lang/lib/Malloy/MalloyParserVisitor.d.ts

@@ -78,6 +78,7 @@ import { JoinWithContext } from "./MalloyParser";
 import { JoinOnContext } from "./MalloyParser";
 import { SQIDContext } from "./MalloyParser";
 import { SQParensContext } from "./MalloyParser";
+import { SQComposeContext } from "./MalloyParser";
 import { SQRefinedQueryContext } from "./MalloyParser";
 import { SQArrowContext } from "./MalloyParser";
 import { SQExtendedSourceContext } from "./MalloyParser";
@@ -785,6 +786,13 @@ export interface MalloyParserVisitor<Result> extends ParseTreeVisitor<Result> {
      * @return the visitor result
      */
     visitSQParens?: (ctx: SQParensContext) => Result;
+    /**
+     * Visit a parse tree produced by the `SQCompose`
+     * labeled alternative in `MalloyParser.sqExpr`.
+     * @param ctx the parse tree
+     * @return the visitor result
+     */
+    visitSQCompose?: (ctx: SQComposeContext) => Result;
     /**
      * Visit a parse tree produced by the `SQRefinedQuery`
      * labeled alternative in `MalloyParser.sqExpr`.

package/dist/lang/malloy-to-ast.d.ts

@@ -212,6 +212,7 @@ export declare class MalloyToAST extends AbstractParseTreeVisitor<ast.MalloyElem
     protected getSqExpr(cx: parse.SqExprContext): ast.SourceQueryElement;
     visitSQExtendedSource(pcx: parse.SQExtendedSourceContext): ast.SQExtend;
     visitSQParens(pcx: parse.SQParensContext): ast.SourceQueryElement;
+    visitSQCompose(pcx: parse.SQComposeContext): ast.SQCompose;
     visitSQArrow(pcx: parse.SQArrowContext): ast.SQArrow;
     getVExpr(pcx: ParserRuleContext): ast.View;
     visitSegField(pcx: parse.SegFieldContext): ast.ReferenceView;

package/dist/lang/malloy-to-ast.js

@@ -1280,6 +1280,11 @@ class MalloyToAST extends AbstractParseTreeVisitor_1.AbstractParseTreeVisitor {
         const sqExpr = this.getSqExpr(pcx.sqExpr());
         return this.astAt(sqExpr, pcx);
     }
+    visitSQCompose(pcx) {
+        const sources = pcx.sqExpr().map(sqExpr => this.getSqExpr(sqExpr));
+        this.inExperiment('composite_sources', pcx);
+        return this.astAt(new ast.SQCompose(sources), pcx);
+    }
     visitSQArrow(pcx) {
         const applyTo = this.getSqExpr(pcx.sqExpr());
         const headCx = pcx.segExpr();

package/dist/lang/parse-log.d.ts

@@ -1,4 +1,4 @@
-import { DocumentLocation, ExpressionValueType } from '../model/malloy_types';
+import { CompositeFieldUsage, DocumentLocation, ExpressionValueType } from '../model/malloy_types';
 import { EventStream } from '../runtime_types';
 export type LogSeverity = 'error' | 'warn' | 'debug';
 /**
@@ -156,6 +156,15 @@ type MessageParameterTypes = {
     'field-list-edit-not-found': string;
     'unexpected-element-type': string;
     'field-not-found': string;
+    'invalid-composite-source-input': string;
+    'invalid-composite-field-usage': {
+        newUsage: CompositeFieldUsage;
+        allUsage: CompositeFieldUsage;
+    };
+    'empty-composite-source': string;
+    'unnecessary-composite-source': string;
+    'composite-source-atomic-fields-only': string;
+    'composite-source-connection-mismatch': string;
     'invalid-property-access-in-field-reference': string;
     'parameter-default-does-not-match-declared-type': string;
     'parameter-null-default-without-declared-type': string;

package/dist/lang/parse-log.js

@@ -23,6 +23,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.makeLogMessage = exports.MESSAGE_FORMATTERS = exports.BaseMessageLogger = void 0;
+const composite_source_utils_1 = require("../model/composite_source_utils");
 class BaseMessageLogger {
     constructor(eventStream) {
         this.eventStream = eventStream;
@@ -94,6 +95,12 @@ exports.MESSAGE_FORMATTERS = {
     'case-else-type-does-not-match': e => `Case else type ${e.elseType} does not match return type ${e.returnType}`,
     'case-when-must-be-boolean': e => `Case when expression must be boolean, not ${e.whenType}`,
     'case-when-type-does-not-match': e => `Case when type ${e.whenType} does not match value type ${e.valueType}`,
+    'invalid-composite-field-usage': e => {
+        const formattedNewCompositeUsage = (0, composite_source_utils_1.formatCompositeFieldUsages)(e.newUsage);
+        const formattedAllCompositeUsage = (0, composite_source_utils_1.formatCompositeFieldUsages)(e.allUsage);
+        const pluralUse = (0, composite_source_utils_1.compositeFieldUsageIsPlural)(e.newUsage) ? 's' : '';
+        return `This operation uses composite field${pluralUse} ${formattedNewCompositeUsage}, resulting in invalid usage of the composite source, as there is no composite input source which defines all of ${formattedAllCompositeUsage}`;
+    },
 };
 function makeLogMessage(code, parameters, options) {
     var _a, _b, _c, _d, _e;

package/dist/lang/test/composite-field-usage.spec.d.ts

@@ -0,0 +1 @@
+import './parse-expects';

package/dist/lang/test/composite-field-usage.spec.js

@@ -0,0 +1,155 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const test_translator_1 = require("./test-translator");
+require("./parse-expects");
+const composite_source_utils_1 = require("../../model/composite_source_utils");
+function addPathToCompositeUsage(path, compositeUsage) {
+    var _a;
+    if (path.length === 0)
+        throw new Error('empty path');
+    if (path.length === 1) {
+        return {
+            fields: [...compositeUsage.fields, path[0]],
+            joinedUsage: compositeUsage.joinedUsage,
+        };
+    }
+    else {
+        return {
+            fields: compositeUsage.fields,
+            joinedUsage: {
+                ...compositeUsage.joinedUsage,
+                [path[0]]: addPathToCompositeUsage(path.slice(1), (_a = compositeUsage.joinedUsage[path[0]]) !== null && _a !== void 0 ? _a : (0, composite_source_utils_1.emptyCompositeFieldUsage)()),
+            },
+        };
+    }
+}
+function paths(paths) {
+    let cu = (0, composite_source_utils_1.emptyCompositeFieldUsage)();
+    for (const path of paths) {
+        cu = addPathToCompositeUsage(path, cu);
+    }
+    return cu;
+}
+describe('composite sources', () => {
+    describe('composite field usage', () => {
+        const m = (0, test_translator_1.model) `
+      ##! experimental.composite_sources
+      source: x is compose(ab, ab) extend {
+        dimension: aif is ai + af
+      }
+
+      source: y is compose(ab, ab) extend {
+        join_one: x on 1 = 1
+      }
+    `;
+        beforeAll(() => {
+            m.translator.translate();
+        });
+        test('looked up value', () => {
+            const mexpr = (0, test_translator_1.makeExprFunc)(m.translator.modelDef, 'y');
+            expect(mexpr `ai`).hasCompositeUsage(paths([['ai']]));
+        });
+        test('multiple values', () => {
+            const mexpr = (0, test_translator_1.makeExprFunc)(m.translator.modelDef, 'y');
+            expect(mexpr `ai + af`).hasCompositeUsage(paths([['ai'], ['af']]));
+        });
+        test('value plus constant', () => {
+            const mexpr = (0, test_translator_1.makeExprFunc)(m.translator.modelDef, 'y');
+            expect(mexpr `ai + 1`).hasCompositeUsage(paths([['ai']]));
+        });
+        test('join usage', () => {
+            const mexpr = (0, test_translator_1.makeExprFunc)(m.translator.modelDef, 'y');
+            expect(mexpr `x.ai + 1`).hasCompositeUsage(paths([['x', 'ai']]));
+        });
+        test('join usage complex', () => {
+            const mexpr = (0, test_translator_1.makeExprFunc)(m.translator.modelDef, 'y');
+            expect(mexpr `x.aif`).hasCompositeUsage(paths([
+                ['x', 'ai'],
+                ['x', 'af'],
+            ]));
+        });
+    });
+    describe('composite source resolution and validation', () => {
+        test('compose fails on group_by that is relevant', () => {
+            expect(`
+        ##! experimental.composite_sources
+        run: compose(
+          a extend { dimension: one is 1, two is 2 },
+          a extend { dimension: one is 1, three is 3 }
+        ) -> {
+          group_by: one
+          group_by: three
+          group_by: ${'two'}
+        }
+      `).toLog((0, test_translator_1.errorMessage)('This operation uses composite field `two`, resulting in invalid usage of the composite source, as there is no composite input source which defines all of `one`, `three`, `two`'));
+        });
+        test('compose fails on filter that is relevant', () => {
+            expect(`
+        ##! experimental.composite_sources
+        run: compose(
+          a extend { dimension: one is 1, two is 2 },
+          a extend { dimension: one is 1, three is 3 }
+        ) -> {
+          group_by: one
+          group_by: three
+          where: ${'two = 2'}
+        }
+      `).toLog((0, test_translator_1.errorMessage)('This operation uses composite field `two`, resulting in invalid usage of the composite source, as there is no composite input source which defines all of `one`, `three`, `two`'));
+        });
+        test('compose fails in case where second field has overlap with first', () => {
+            expect(`
+        ##! experimental.composite_sources
+        run: compose(
+          a extend { dimension: one is 1 },
+          a extend { dimension: two is 3 }
+        ) -> {
+          group_by: one
+          group_by: ${'three is one + two'}
+        }
+      `).toLog((0, test_translator_1.errorMessage)('This operation uses composite field `two`, resulting in invalid usage of the composite source, as there is no composite input source which defines all of `one`, `two`'));
+        });
+        test('compose resolution succeeds nested', () => {
+            expect(`
+        ##! experimental.composite_sources
+        run: compose(
+          compose(
+            a extend { dimension: one is 1, two is 2 },
+            a extend { dimension: one is 1, three is 3 }
+          ),
+          a extend { dimension: one is 1, two is 2, three is 3 }
+        ) -> {
+          group_by: one, two ${'three'}
+        }
+      `).toTranslate();
+        });
+        test('good composite field usage works', () => expect(`
+        ##! experimental.composite_sources
+        run: compose(a, a extend { dimension: one is 1 }) -> { group_by: one }
+    `).toTranslate());
+        test.skip('index on composite fails', () => expect(`
+        ##! experimental.composite_sources
+        run: compose(a extend { dimension: two is 2 }, a extend { dimension: one is 1 }) -> { index: * }
+    `).toLog((0, test_translator_1.errorMessage)('Cannot index on a composite source')));
+        test('raw run of composite source fails', () => expect(`
+        ##! experimental.composite_sources
+        run: compose(a extend { dimension: two is 2 }, a extend { dimension: one is 1 })
+    `).toLog((0, test_translator_1.errorMessage)('Cannot run this object as a query')));
+        test('composite source with parameter', () => {
+            expect(`
+        ##! experimental { composite_sources parameters }
+        source: foo(param is 1) is compose(
+          a extend { dimension: x is param },
+          a extend { dimension: y is param + 1 }
+        )
+        run: foo(param is 2) -> { group_by: y }
+      `).toTranslate();
+        });
+    });
+});
+//# sourceMappingURL=composite-field-usage.spec.js.map

package/dist/lang/test/parse-expects.d.ts

@@ -1,4 +1,4 @@
-import { DocumentLocation } from '../../model';
+import { CompositeFieldUsage, DocumentLocation } from '../../model';
 import { LogSeverity } from '../parse-log';
 type MessageProblemSpec = {
     severity: LogSeverity;
@@ -45,6 +45,7 @@ declare global {
              * Warnings are ignored, so need to be checked seperately
              */
             compilesTo(exprString: string): R;
+            hasCompositeUsage(compositeUsage: CompositeFieldUsage): R;
         }
     }
 }

package/dist/lang/test/parse-expects.js

@@ -276,6 +276,42 @@ expect.extend({
         const msg = pass ? `Matched: ${rcvExpr}` : this.utils.diff(expr, rcvExpr);
         return { pass, message: () => `${msg}` };
     },
+    hasCompositeUsage: function (tx, compositeFieldUsage) {
+        let bx;
+        if (typeof tx === 'string') {
+            bx = new test_translator_1.BetaExpression(tx);
+        }
+        else {
+            const x = xlator(tx);
+            if (x instanceof test_translator_1.BetaExpression) {
+                bx = x;
+            }
+            else {
+                return {
+                    pass: false,
+                    message: () => 'Must pass expr`EXPRESSION` to expect(EXPRSSION).compilesTo()',
+                };
+            }
+        }
+        bx.compile();
+        // Only report errors, callers will need to test for warnings
+        if (bx.logger.hasErrors()) {
+            return {
+                message: () => `Translation problems:\n${bx.prettyErrors()}`,
+                pass: false,
+            };
+        }
+        const badRefs = checkForNeededs(bx);
+        if (!badRefs.pass) {
+            return badRefs;
+        }
+        const actual = bx.generated().compositeFieldUsage;
+        const pass = this.equals(actual, compositeFieldUsage);
+        const msg = pass
+            ? `Matched: ${actual}`
+            : this.utils.diff(compositeFieldUsage, actual);
+        return { pass, message: () => `${msg}` };
+    },
 });
 function problemSpecSummary(s) {
     return `${s.severity} '${'message' in s ? s.message : s.code}' ${s.data !== undefined ? (0, test_translator_1.pretty)(s.data) : ''}`;

package/dist/lang/test/source.spec.js

@@ -316,5 +316,14 @@ describe('source:', () => {
         const ad = (0, test_translator_1.getFieldDef)(hasDate, 'ad');
         expect(ad).toMatchObject({ name: 'ad', type: 'date' });
     });
+    describe('composite sources', () => {
+        test('basic composite source', () => {
+            expect(`
+        ##! experimental.composite_sources
+        source: x is compose(a, a extend { dimension: foo is 1 })
+        run: x -> { select: foo }
+      `).toTranslate();
+        });
+    });
 });
 //# sourceMappingURL=source.spec.js.map

package/dist/lang/test/test-translator.d.ts

@@ -45,8 +45,9 @@ export declare class TestTranslator extends MalloyTranslator {
     getQuery(queryName: string | number): Query | undefined;
 }
 export declare class BetaExpression extends TestTranslator {
+    readonly sourceName: string;
     private compiled?;
-    constructor(src: string);
+    constructor(src: string, model?: ModelDef, sourceName?: string);
     private testFS;
     compile(): void;
     generated(): ExprValue;
@@ -72,6 +73,7 @@ export declare function makeModelFunc(options: {
     prefix?: string;
     wrap?: (code: string) => string;
 }): (unmarked: TemplateStringsArray, ...marked: string[]) => HasTranslator<TestTranslator>;
+export declare function makeExprFunc(model: ModelDef, sourceName: string): (unmarked: TemplateStringsArray, ...marked: string[]) => HasTranslator<TestTranslator>;
 export declare function markSource(unmarked: TemplateStringsArray, ...marked: string[]): MarkedSource;
 export declare function getSelectOneStruct(sqlBlock: SQLSentence): SQLSourceDef;
 export declare function error<T extends MessageCode>(code: T, data?: MessageParameterType<T>): {

package/dist/lang/test/test-translator.js

@@ -23,7 +23,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.warningMessage = exports.errorMessage = exports.warning = exports.error = exports.getSelectOneStruct = exports.markSource = exports.makeModelFunc = exports.model = exports.expr = exports.getJoinField = exports.getQueryField = exports.getQueryFieldDef = exports.getFieldDef = exports.getModelQuery = exports.getExplore = exports.BetaExpression = exports.TestTranslator = exports.TestChildTranslator = exports.aTableDef = exports.pretty = void 0;
+exports.warningMessage = exports.errorMessage = exports.warning = exports.error = exports.getSelectOneStruct = exports.markSource = exports.makeExprFunc = exports.makeModelFunc = exports.model = exports.expr = exports.getJoinField = exports.getQueryField = exports.getQueryFieldDef = exports.getFieldDef = exports.getModelQuery = exports.getExplore = exports.BetaExpression = exports.TestTranslator = exports.TestChildTranslator = exports.aTableDef = exports.pretty = void 0;
 const util_1 = require("util");
 const malloy_types_1 = require("../../model/malloy_types");
 const ast_1 = require("../ast");
@@ -386,11 +386,12 @@ class TestTranslator extends parse_malloy_1.MalloyTranslator {
 exports.TestTranslator = TestTranslator;
 TestTranslator.inspectCompile = false;
 class BetaExpression extends TestTranslator {
-    constructor(src) {
-        super(src, null, null, 'debugExpr');
+    constructor(src, model, sourceName = 'ab') {
+        super(src, null, null, 'debugExpr', model);
+        this.sourceName = sourceName;
     }
     testFS() {
-        const aStruct = this.internalModel.contents['ab'];
+        const aStruct = this.internalModel.contents[this.sourceName];
         if ((0, malloy_types_1.isSourceDef)(aStruct)) {
             const tstFS = new static_space_1.StaticSourceSpace(aStruct);
             return tstFS;
@@ -498,6 +499,16 @@ function makeModelFunc(options) {
     };
 }
 exports.makeModelFunc = makeModelFunc;
+function makeExprFunc(model, sourceName) {
+    return function expr(unmarked, ...marked) {
+        const ms = markSource(unmarked, ...marked);
+        return {
+            ...ms,
+            translator: new BetaExpression(ms.code, model, sourceName),
+        };
+    };
+}
+exports.makeExprFunc = makeExprFunc;
 function markSource(unmarked, ...marked) {
     let code = '';
     const locations = [];

package/dist/lang/utils.d.ts

@@ -1,2 +1,3 @@
 import { DocumentLocation, DocumentPosition } from '../model/malloy_types';
 export declare function locationContainsPosition(location: DocumentLocation, position: DocumentPosition): boolean;
+export declare function isNotUndefined<T>(value: T | undefined): value is T;

package/dist/lang/utils.js

@@ -22,7 +22,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.locationContainsPosition = void 0;
+exports.isNotUndefined = exports.locationContainsPosition = void 0;
 function locationContainsPosition(location, position) {
     return (location.range.start.line <= position.line &&
         location.range.end.line >= position.line &&
@@ -32,4 +32,8 @@ function locationContainsPosition(location, position) {
             position.character <= location.range.end.character));
 }
 exports.locationContainsPosition = locationContainsPosition;
+function isNotUndefined(value) {
+    return value !== undefined;
+}
+exports.isNotUndefined = isNotUndefined;
 //# sourceMappingURL=utils.js.map

package/dist/model/composite_source_utils.d.ts

@@ -0,0 +1,65 @@
+import { CompositeFieldUsage, SourceDef } from './malloy_types';
+type CompositeError = {
+    code: 'not_a_composite_source';
+    data: {
+        path: string[];
+    };
+} | {
+    code: 'composite_source_not_defined';
+    data: {
+        path: string[];
+    };
+} | {
+    code: 'composite_source_not_a_join';
+    data: {
+        path: string[];
+    };
+} | {
+    code: 'composite_source_is_not_joinable';
+    data: {
+        path: string[];
+    };
+} | {
+    code: 'no_suitable_composite_source_input';
+    data: {
+        path: string[];
+        fields: string[];
+    };
+};
+type SingleNarrowedCompositeFieldResolution = {
+    source: SourceDef;
+    nested?: SingleNarrowedCompositeFieldResolution | undefined;
+}[];
+interface NarrowedCompositeFieldResolution {
+    source: SingleNarrowedCompositeFieldResolution | undefined;
+    joined: {
+        [name: string]: NarrowedCompositeFieldResolution;
+    };
+}
+export declare function emptyNarrowedCompositeFieldResolution(): NarrowedCompositeFieldResolution;
+export declare function narrowCompositeFieldResolution(source: SourceDef, compositeFieldUsage: CompositeFieldUsage, narrowedCompositeFieldResolution: NarrowedCompositeFieldResolution): {
+    narrowedCompositeFieldResolution: NarrowedCompositeFieldResolution;
+    error: undefined;
+} | {
+    error: CompositeError;
+    narrowedCompositeFieldResolution: undefined;
+};
+export declare function resolveCompositeSources(source: SourceDef, compositeFieldUsage: CompositeFieldUsage): {
+    sourceDef: SourceDef;
+    error: undefined;
+} | {
+    error: CompositeError;
+    sourceDef: undefined;
+};
+export declare function compositeFieldUsagePaths(compositeFieldUsage: CompositeFieldUsage): string[][];
+export declare function formatCompositeFieldUsages(compositeFieldUsage: CompositeFieldUsage): string;
+export declare function isEmptyCompositeFieldUsage(compositeFieldUsage: CompositeFieldUsage): boolean;
+export declare function compositeFieldUsageIsPlural(compositeFieldUsage: CompositeFieldUsage): boolean;
+export declare function formatCompositeFieldUsage(compositeFieldUsage: string[]): string;
+export declare function unique<T>(values: T[]): T[];
+export declare function mergeCompositeFieldUsage(...usages: (CompositeFieldUsage | undefined)[]): CompositeFieldUsage;
+export declare function emptyCompositeFieldUsage(): CompositeFieldUsage;
+export declare function compositeFieldUsageDifference(a: CompositeFieldUsage, b: CompositeFieldUsage): CompositeFieldUsage;
+export declare function joinedCompositeFieldUsage(joinPath: string[], compositeFieldUsage: CompositeFieldUsage): CompositeFieldUsage;
+export declare function compositeFieldUsageJoinPaths(compositeFieldUsage: CompositeFieldUsage): string[][];
+export {};