@malloydata/malloy 0.0.240-dev250311202829 → 0.0.240-dev250311213218

package/dist/lang/malloy-to-stable-query.js

@@ -0,0 +1,660 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.malloyToQuery = exports.MalloyToQuery = void 0;
+const AbstractParseTreeVisitor_1 = require("antlr4ts/tree/AbstractParseTreeVisitor");
+const parse = __importStar(require("./lib/Malloy/MalloyParser"));
+const parse_log_1 = require("./parse-log");
+const parse_utils_1 = require("./parse-utils");
+const malloy_types_1 = require("../model/malloy_types");
+const run_malloy_parser_1 = require("./run-malloy-parser");
+const core_1 = require("../api/core");
+const utils_1 = require("./utils");
+const MLQs = 'Malloy query documents';
+/**
+ * ANTLR visitor pattern parse tree traversal. Generates a Malloy
+ * AST from an ANTLR parse tree.
+ */
+class MalloyToQuery extends AbstractParseTreeVisitor_1.AbstractParseTreeVisitor {
+    constructor(parseInfo, msgLog) {
+        super();
+        this.parseInfo = parseInfo;
+        this.msgLog = msgLog;
+    }
+    /**
+     * Mostly used to flag a case where the grammar and the type system are
+     * no longer in sync. A visitor was written based on a grammar which
+     * apparently has changed and now an unexpected element type has appeared.
+     * This is a non recoverable error, since the parser and the grammar
+     * are not compatible.
+     * @return an error object to throw.
+     */
+    internalError(cx, message) {
+        this.contextError(cx, 'internal-translator-error', { message });
+        return new Error(`Internal Translator Error: ${message}`);
+    }
+    getLocation(cx) {
+        return {
+            url: this.parseInfo.sourceURL,
+            range: (0, utils_1.rangeFromContext)(this.parseInfo.sourceInfo, cx),
+        };
+    }
+    /**
+     * Log an error message relative to a parse node
+     */
+    contextError(cx, code, data, options) {
+        this.msgLog.log((0, parse_log_1.makeLogMessage)(code, data, {
+            at: this.getLocation(cx),
+            ...options,
+        }));
+    }
+    getNumber(term) {
+        return Number.parseInt(term.text);
+    }
+    defaultResult() {
+        return null;
+    }
+    /**
+     * Get all the possibly missing annotations from this parse rule
+     * @param cx Any parse context which has an ANNOTATION* rules
+     * @returns Array of texts for the annotations
+     */
+    getAnnotations(cx) {
+        const annotations = cx.ANNOTATION().map(a => {
+            return { value: a.text };
+        });
+        return annotations.length > 0 ? annotations : undefined;
+    }
+    getIsAnnotations(cx) {
+        var _a, _b;
+        if (cx === undefined)
+            return undefined;
+        const before = (_a = this.getAnnotations(cx._beforeIs)) !== null && _a !== void 0 ? _a : [];
+        const annotations = before.concat((_b = this.getAnnotations(cx._afterIs)) !== null && _b !== void 0 ? _b : []);
+        return annotations.length > 0 ? annotations : undefined;
+    }
+    notAllowed(pcx, what) {
+        this.illegal(pcx, `${what} are not allowed in ${MLQs}`);
+    }
+    illegal(pcx, what) {
+        this.contextError(pcx, 'invalid-malloy-query-document', what);
+    }
+    visitMalloyDocument(pcx) {
+        const statements = pcx.malloyStatement();
+        let runStatement = undefined;
+        for (const statement of statements) {
+            if (statement.defineSourceStatement()) {
+                this.notAllowed(statement, 'Source definitions');
+            }
+            else if (statement.defineQuery()) {
+                this.notAllowed(statement, 'Query definitions');
+            }
+            else if (statement.importStatement()) {
+                this.notAllowed(statement, 'Import statements');
+            }
+            else if (statement.docAnnotations()) {
+                this.notAllowed(statement, 'Model annotations');
+            }
+            else if (statement.ignoredObjectAnnotations()) {
+                this.notAllowed(statement, 'Detatched object annotations');
+            }
+            else if (statement.experimentalStatementForTesting()) {
+                this.notAllowed(statement, 'Experimental testing statements');
+            }
+            else {
+                if (runStatement === undefined) {
+                    runStatement = statement.runStatement();
+                }
+                else {
+                    this.illegal(statement, `${MLQs} may only have one run statement`);
+                }
+            }
+        }
+        if (runStatement === undefined) {
+            this.illegal(pcx, `${MLQs} must have a run statement`);
+            return null;
+        }
+        return this.visitRunStatement(runStatement);
+    }
+    visitRunStatement(pcx) {
+        const defCx = pcx.topLevelAnonQueryDef();
+        const definition = this.getQueryDefinition(defCx.sqExpr());
+        const annotations = this.getAnnotations(pcx.topLevelAnonQueryDef().tags());
+        if (definition !== null) {
+            return {
+                annotations,
+                definition,
+            };
+        }
+        return null;
+    }
+    getQueryReference(cx) {
+        if (cx.sourceArguments()) {
+            this.illegal(cx, 'Queries do not support parameters');
+        }
+        else {
+            return {
+                name: (0, parse_utils_1.getId)(cx),
+            };
+        }
+        return null;
+    }
+    getQueryDefinition(cx) {
+        if (cx instanceof parse.SQIDContext) {
+            const ref = this.getQueryReference(cx);
+            if (ref !== null) {
+                return {
+                    kind: 'query_reference',
+                    ...ref,
+                };
+            }
+        }
+        else if (cx instanceof parse.SQParensContext) {
+            return this.getQueryDefinition(cx.sqExpr());
+        }
+        else if (cx instanceof parse.SQComposeContext) {
+            this.notAllowed(cx, 'Source compositions');
+        }
+        else if (cx instanceof parse.SQRefinedQueryContext) {
+            const qrefCx = cx.sqExpr();
+            const base = this.getQueryDefinition(qrefCx);
+            const seg = this.getRefinementSegment(cx.segExpr());
+            if (seg === null || base === null)
+                return null;
+            if (seg.kind === 'arrow') {
+                this.notAllowed(cx, 'Queries against refined queries');
+            }
+            return {
+                kind: 'refinement',
+                base: base,
+                refinement: seg,
+            };
+        }
+        else if (cx instanceof parse.SQExtendedSourceContext) {
+            this.notAllowed(cx, 'Source extensions');
+        }
+        else if (cx instanceof parse.SQIncludeContext) {
+            this.notAllowed(cx, 'Source inclusions');
+        }
+        else if (cx instanceof parse.SQTableContext) {
+            this.notAllowed(cx, 'Table statements');
+        }
+        else if (cx instanceof parse.SQSQLContext) {
+            this.notAllowed(cx, 'SQL statements');
+        }
+        else if (cx instanceof parse.SQArrowContext) {
+            const qrefCx = cx.sqExpr();
+            const base = this.getQueryDefinition(qrefCx);
+            const seg = this.getRefinementSegment(cx.segExpr());
+            if (seg === null || base === null)
+                return null;
+            if (base.kind === 'query_reference') {
+                return {
+                    kind: 'arrow',
+                    source: {
+                        ...base,
+                        kind: 'source_reference',
+                    },
+                    view: seg,
+                };
+            }
+            if (base.kind === 'arrow') {
+                return {
+                    kind: 'arrow',
+                    source: base.source,
+                    view: {
+                        kind: 'arrow',
+                        source: base.view,
+                        view: seg,
+                    },
+                };
+            }
+            return {
+                kind: 'arrow',
+                source: base,
+                view: seg,
+            };
+        }
+        return null;
+    }
+    getRefinementSegment(cx) {
+        if (cx instanceof parse.SegOpsContext) {
+            const operations = cx
+                .queryProperties()
+                .queryStatement()
+                .flatMap(stmt => this.getSegmentOperation(stmt));
+            if (operations.some(o => o === null)) {
+                return null;
+            }
+            return {
+                kind: 'segment',
+                operations: operations,
+            };
+        }
+        else if (cx instanceof parse.SegFieldContext) {
+            const { name, path } = this.getFieldPath(cx.fieldPath());
+            return {
+                kind: 'view_reference',
+                name,
+                path,
+            };
+        }
+        else if (cx instanceof parse.SegParenContext) {
+            return this.getViewExpression(cx.vExpr());
+        }
+        else if (cx instanceof parse.SegRefineContext) {
+            const l = this.getRefinementSegment(cx._lhs);
+            const r = this.getRefinementSegment(cx._rhs);
+            if (l === null || r === null)
+                return null;
+            return {
+                kind: 'refinement',
+                base: l,
+                refinement: r,
+            };
+        }
+        return null;
+    }
+    getGroupByStatement(gbcx) {
+        const groupAnnotations = this.getAnnotations(gbcx.tags());
+        const fieldCxs = gbcx.queryFieldList().queryFieldEntry();
+        const fields = fieldCxs.map(f => this.getQueryField(f));
+        if (fields.some(o => o === null)) {
+            return null;
+        }
+        if (fields === null)
+            return null;
+        return fields.map(f => {
+            var _a;
+            const annotations = [
+                ...(groupAnnotations !== null && groupAnnotations !== void 0 ? groupAnnotations : []),
+                ...((_a = f.field.annotations) !== null && _a !== void 0 ? _a : []),
+            ];
+            const gb = {
+                kind: 'group_by',
+                name: f.name,
+                field: {
+                    ...f.field,
+                    annotations: annotations.length > 0 ? annotations : undefined,
+                },
+            };
+            return gb;
+        });
+    }
+    getAggregateStatement(agcx) {
+        const groupAnnotations = this.getAnnotations(agcx.tags());
+        const fieldCxs = agcx.queryFieldList().queryFieldEntry();
+        const fields = fieldCxs.map(f => this.getQueryField(f));
+        if (fields.some(o => o === null)) {
+            return null;
+        }
+        if (fields === null)
+            return null;
+        return fields.map(f => {
+            var _a;
+            const annotations = [
+                ...(groupAnnotations !== null && groupAnnotations !== void 0 ? groupAnnotations : []),
+                ...((_a = f.field.annotations) !== null && _a !== void 0 ? _a : []),
+            ];
+            const gb = {
+                kind: 'aggregate',
+                name: f.name,
+                field: {
+                    ...f.field,
+                    annotations: annotations.length > 0 ? annotations : undefined,
+                },
+            };
+            return gb;
+        });
+    }
+    getOrderByStatement(obcx) {
+        const specs = obcx.ordering().orderBySpec();
+        const orders = [];
+        for (const spec of specs) {
+            if (spec.INTEGER_LITERAL()) {
+                this.notAllowed(spec, 'Indexed order by statements');
+            }
+            else if (spec.fieldName()) {
+                const fieldName = (0, parse_utils_1.getId)(spec.fieldName());
+                const direction = spec.ASC() ? 'asc' : spec.DESC() ? 'desc' : undefined;
+                orders.push({
+                    kind: 'order_by',
+                    direction,
+                    field_reference: { name: fieldName },
+                });
+            }
+            else {
+                return null;
+            }
+        }
+        return orders;
+    }
+    getNestStatement(nstcx) {
+        const groupAnnotations = this.getAnnotations(nstcx.tags());
+        const querylist = nstcx.nestedQueryList().nestEntry();
+        const nests = [];
+        for (const query of querylist) {
+            if (!(query instanceof parse.NestDefContext)) {
+                this.internalError(query, 'Expected nestDef');
+                return null;
+            }
+            const annotations1 = this.getAnnotations(query.tags());
+            const annotations2 = this.getIsAnnotations(query.isDefine());
+            const nameCx = query.queryName();
+            const name = nameCx ? (0, parse_utils_1.getId)(nameCx) : undefined;
+            const view = this.getViewExpression(query.vExpr());
+            if (view === null) {
+                return null;
+            }
+            nests.push({
+                kind: 'nest',
+                name,
+                view: {
+                    definition: view,
+                    annotations: this.combineAnnotations(groupAnnotations, annotations1, annotations2),
+                },
+            });
+        }
+        return nests;
+    }
+    getViewExpression(cx) {
+        if (cx instanceof parse.VSegContext) {
+            return this.getRefinementSegment(cx.segExpr());
+        }
+        else if (cx instanceof parse.VArrowContext) {
+            const l = this.getRefinementSegment(cx);
+            const r = this.getViewExpression(cx._rhs);
+            if (l === null)
+                return null;
+            if (r === null)
+                return null;
+            return {
+                kind: 'arrow',
+                source: l,
+                view: r,
+            };
+        }
+        else {
+            this.internalError(cx, 'Unexpected VExpr node');
+            return null;
+        }
+    }
+    getLimitStatement(cx) {
+        const limit = this.getNumber(cx.INTEGER_LITERAL());
+        return {
+            kind: 'limit',
+            limit,
+        };
+    }
+    getSegmentOperation(cx) {
+        if (cx.groupByStatement()) {
+            const gbcx = cx.groupByStatement();
+            return this.getGroupByStatement(gbcx);
+        }
+        else if (cx.aggregateStatement()) {
+            const agcx = cx.aggregateStatement();
+            return this.getAggregateStatement(agcx);
+        }
+        else if (cx.limitStatement()) {
+            const limcx = cx.limitStatement();
+            const limit = this.getLimitStatement(limcx);
+            if (limit === null)
+                return null;
+            return [limit];
+        }
+        else if (cx.declareStatement()) {
+            this.notAllowed(cx, 'Declare statements');
+        }
+        else if (cx.queryJoinStatement()) {
+            this.notAllowed(cx, 'Query join statements');
+        }
+        else if (cx.queryExtend()) {
+            this.notAllowed(cx, 'Query extend statements');
+        }
+        else if (cx.projectStatement()) {
+            this.notAllowed(cx, 'Select statements');
+        }
+        else if (cx.indexStatement()) {
+            this.notAllowed(cx, 'Index statements');
+        }
+        else if (cx.calculateStatement()) {
+            this.notAllowed(cx, 'Calculate statements');
+        }
+        else if (cx.topStatement()) {
+            this.notAllowed(cx, 'Top statements');
+        }
+        else if (cx.orderByStatement()) {
+            const obcx = cx.orderByStatement();
+            return this.getOrderByStatement(obcx);
+        }
+        else if (cx.whereStatement()) {
+            const whcx = cx.whereStatement();
+            const where = this.getWhere(whcx);
+            if (where === null)
+                return null;
+            return where.map(w => ({
+                kind: 'where',
+                ...w,
+            }));
+        }
+        else if (cx.havingStatement()) {
+            this.notAllowed(cx, 'Having statements');
+        }
+        else if (cx.nestStatement()) {
+            const obcx = cx.nestStatement();
+            return this.getNestStatement(obcx);
+        }
+        else if (cx.sampleStatement()) {
+            this.notAllowed(cx, 'Sample statements');
+        }
+        else if (cx.timezoneStatement()) {
+            this.notAllowed(cx, 'Timezone statements');
+        }
+        else if (cx.queryAnnotation() || cx.ignoredModelAnnotations()) {
+            this.notAllowed(cx, 'Detached annotation statements');
+        }
+        return null;
+    }
+    getFieldPath(pcx) {
+        const names = pcx.fieldName().map(nameCx => (0, parse_utils_1.getId)(nameCx));
+        const name = names[0];
+        const path = names.slice(1);
+        return { name, path: path.length > 0 ? path : undefined };
+    }
+    getTimeframe(cx) {
+        const text = cx.text;
+        if ((0, malloy_types_1.isTimestampUnit)(text)) {
+            return text;
+        }
+        this.illegal(cx, `Invalid timeframe ${text}`);
+        return null;
+    }
+    getQueryField(cx) {
+        if (cx.taggedRef()) {
+            const trcx = cx.taggedRef();
+            const annotations = this.getAnnotations(trcx.tags());
+            const { name, path } = this.getFieldPath(trcx.fieldPath());
+            if (trcx.refExpr()) {
+                const refexpr = trcx.refExpr();
+                if (refexpr.timeframe()) {
+                    const tf = this.getTimeframe(refexpr.timeframe());
+                    if (tf === null)
+                        return null;
+                    return {
+                        name: undefined,
+                        field: {
+                            annotations,
+                            expression: {
+                                kind: 'time_truncation',
+                                field_reference: {
+                                    name,
+                                    path,
+                                },
+                                truncation: tf,
+                            },
+                        },
+                    };
+                }
+                else if (refexpr.aggregate()) {
+                    this.notAllowed(refexpr, 'Aggregate expressions');
+                }
+            }
+            else {
+                return {
+                    name: undefined,
+                    field: {
+                        annotations,
+                        expression: {
+                            kind: 'field_reference',
+                            name,
+                            path,
+                        },
+                    },
+                };
+            }
+        }
+        else if (cx.fieldDef()) {
+            const defCx = cx.fieldDef();
+            const annotations1 = this.getAnnotations(defCx.tags());
+            const annotations2 = this.getIsAnnotations(defCx.isDefine());
+            const name = (0, parse_utils_1.getId)(defCx.fieldNameDef());
+            const expression = this.getFieldExpression(defCx.fieldExpr());
+            if (expression === null) {
+                return null;
+            }
+            return {
+                name,
+                field: {
+                    expression,
+                    annotations: this.combineAnnotations(annotations1, annotations2),
+                },
+            };
+        }
+        return null;
+    }
+    getFieldExpression(cx) {
+        if (cx instanceof parse.ExprFieldPathContext) {
+            const { name, path } = this.getFieldPath(cx.fieldPath());
+            return {
+                kind: 'field_reference',
+                name,
+                path,
+            };
+        }
+        else if (cx instanceof parse.ExprTimeTruncContext) {
+            const timeframe = this.getTimeframe(cx.timeframe());
+            const exprCx = cx.fieldExpr();
+            const expr = this.getFieldExpression(exprCx);
+            if (expr === null) {
+                return null;
+            }
+            if (timeframe === null) {
+                return null;
+            }
+            if (expr.kind !== 'field_reference') {
+                this.illegal(exprCx, 'Left hand side of time truncation must be a field reference');
+                return null;
+            }
+            return {
+                kind: 'time_truncation',
+                truncation: timeframe,
+                field_reference: {
+                    name: expr.name,
+                    path: expr.path,
+                    parameters: expr.parameters,
+                },
+            };
+        }
+        else if (cx instanceof parse.ExprFieldPropsContext) {
+            const exprCx = cx.fieldExpr();
+            const expr = this.getFieldExpression(exprCx);
+            if (expr === null) {
+                return null;
+            }
+            if (expr.kind !== 'field_reference') {
+                this.illegal(exprCx, 'Left hand side of filtered field must be a field reference');
+                return null;
+            }
+            const props = cx.fieldProperties().fieldPropertyStatement();
+            const where = [];
+            for (const prop of props) {
+                const whereCx = prop.whereStatement();
+                if (whereCx) {
+                    const it = this.getWhere(whereCx);
+                    if (it === null)
+                        return null;
+                    where.push(...it);
+                }
+            }
+            return {
+                kind: 'filtered_field',
+                field_reference: {
+                    name: expr.name,
+                    path: expr.path,
+                    parameters: expr.parameters,
+                },
+                where,
+            };
+        }
+        return null;
+    }
+    getWhere(_whereCx) {
+        // const exprs = whereCx.filterClauseList().fieldExpr();
+        // TODO get filter expr...
+        return null;
+    }
+    combineAnnotations(...a) {
+        const annotations = a.flatMap(a => a !== null && a !== void 0 ? a : []);
+        return annotations.length > 0 ? annotations : undefined;
+    }
+}
+exports.MalloyToQuery = MalloyToQuery;
+function malloyToQuery(code) {
+    const sourceInfo = (0, utils_1.getSourceInfo)(code);
+    const logger = new parse_log_1.BaseMessageLogger(null);
+    const url = 'internal://query.malloy';
+    const parse = (0, run_malloy_parser_1.runMalloyParser)(code, url, sourceInfo, logger);
+    const secondPass = new MalloyToQuery(parse, logger);
+    const query = secondPass.visit(parse.root);
+    const logs = (0, core_1.mapLogs)(logger.getLog(), url);
+    if (query === null) {
+        return { logs };
+    }
+    if (!('definition' in query)) {
+        throw new Error('Expected a query');
+    }
+    return {
+        query: query,
+        logs,
+    };
+}
+exports.malloyToQuery = malloyToQuery;
+//# sourceMappingURL=malloy-to-stable-query.js.map

package/dist/lang/parse-log.d.ts

@@ -362,6 +362,7 @@ type MessageParameterTypes = {
     'unsupported-path-in-include': string;
     'wildcard-include-rename': string;
     'literal-string-newline': string;
+    'invalid-malloy-query-document': string;
 };
 export declare const MESSAGE_FORMATTERS: PartialErrorCodeMessageMap;
 export type MessageCode = keyof MessageParameterTypes;

package/dist/lang/parse-malloy.d.ts

@@ -1,5 +1,4 @@
-import { ParserRuleContext, Token } from 'antlr4ts';
-import { DocumentLocation, DocumentPosition, DocumentRange, DocumentReference, ImportLocation, ModelDef, NamedModelObject, SourceDef, SQLSourceDef, DependencyTree } from '../model/malloy_types';
+import { DocumentLocation, DocumentPosition, DocumentReference, ImportLocation, ModelDef, NamedModelObject, SourceDef, SQLSourceDef, DependencyTree, DocumentRange } from '../model/malloy_types';
 import { BaseMessageLogger, LogMessage, LogMessageOptions, MessageCode, MessageParameterType } from './parse-log';
 import { Zone, ZoneData } from './zone';
 import { ReferenceList } from './reference-list';
@@ -7,6 +6,7 @@ import { ASTResponse, CompletionsResponse, DataRequestResponse, ProblemResponse,
 import { Tag } from '@malloydata/malloy-tag';
 import { MalloyParseInfo } from './malloy-parse-info';
 import { EventStream } from '../runtime_types';
+import { ParserRuleContext } from 'antlr4ts';
 export type StepResponses = DataRequestResponse | ASTResponse | TranslateResponse | ParseResponse | MetadataResponse | PretranslatedResponse;
 /**
  * A Translation is a series of translation steps. Each step can depend
@@ -46,11 +46,6 @@ declare class ParseStep implements TranslationStep {
     response?: ParseResponse;
     sourceInfo?: SourceInfo;
     step(that: MalloyTranslation): ParseResponse;
-    /**
-     * Split the source up into lines so we can correctly compute offset
-     * to the line/char positions favored by LSP and VSCode.
-     */
-    private getSourceInfo;
     private runParser;
 }
 declare class ImportsAndTablesStep implements TranslationStep {
@@ -173,8 +168,6 @@ export declare abstract class MalloyTranslation {
     }): HelpContextResponse;
     defaultLocation(): DocumentLocation;
     rangeFromContext(pcx: ParserRuleContext): DocumentRange;
-    rangeFromTokens(startToken: Token, stopToken: Token): DocumentRange;
-    rangeFromToken(token: Token): DocumentRange;
     private dialectAlreadyChecked;
     firstReferenceToDialect(dialect: string): boolean;
     allDialectsEnabled: boolean;