@malloydata/malloy 0.0.1

package/dist/lang/field-space.js

@@ -0,0 +1,651 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fsPair = exports.DefSpace = exports.IndexFieldSpace = exports.ProjectFieldSpace = exports.ReduceFieldSpace = exports.QuerySpace = exports.QueryOperationSpace = exports.DynamicSpace = exports.StaticSpace = void 0;
+/* eslint-disable no-useless-constructor */
+const lodash_1 = require("lodash");
+const dialect_1 = require("../dialect");
+const model = __importStar(require("../model/malloy_types"));
+const ast_1 = require("./ast");
+const space_field_1 = require("./space-field");
+const field_utils_1 = require("./field-utils");
+/**
+ * The father of all FieldSpaces is a wrapper for a StructDef.
+ */
+class StaticSpace {
+    constructor(sourceStructDef) {
+        this.type = "fieldSpace";
+        this.fromStruct = sourceStructDef;
+    }
+    whenComplete(step) {
+        step();
+    }
+    getDialect() {
+        return (0, dialect_1.getDialect)(this.fromStruct.dialect);
+    }
+    fromFieldDef(from) {
+        if (from.type === "struct") {
+            return new space_field_1.StructSpaceField(from);
+        }
+        else if (model.isTurtleDef(from)) {
+            return new space_field_1.QueryFieldStruct(this, from);
+        }
+        // TODO field has a filter and needs to be a filteredalias
+        return new space_field_1.ColumnSpaceField(from);
+    }
+    get map() {
+        if (this.memoMap === undefined) {
+            this.memoMap = {};
+            for (const f of this.fromStruct.fields) {
+                const name = f.as || f.name;
+                this.memoMap[name] = this.fromFieldDef(f);
+            }
+            if (this.fromStruct.parameters) {
+                for (const [paramName, paramDef] of Object.entries(this.fromStruct.parameters)) {
+                    if (this.memoMap[paramName]) {
+                        throw new Error(`In struct '${this.fromStruct.as || this.fromStruct.name}': ` +
+                            ` : Field and parameter name conflict '${paramName}`);
+                    }
+                    this.memoMap[paramName] = new space_field_1.DefinedParameter(paramDef);
+                }
+            }
+        }
+        return this.memoMap;
+    }
+    dropEntries() {
+        this.memoMap = {};
+    }
+    dropEntry(name) {
+        delete this.map[name];
+    }
+    entry(name) {
+        return this.map[name];
+    }
+    setEntry(name, value) {
+        this.map[name] = value;
+    }
+    entries() {
+        return Object.entries(this.map);
+    }
+    structDef() {
+        return this.fromStruct;
+    }
+    emptyStructDef() {
+        return { ...this.fromStruct, fields: [] };
+    }
+    lookup(path) {
+        const head = path[0];
+        const rest = path.slice(1);
+        const found = this.entry(head.refString);
+        if (!found) {
+            return { error: `'${head}' is not defined`, found };
+        }
+        if (found instanceof space_field_1.SpaceField) {
+            /*
+             * In the cleanup phase of query space construction, it may check
+             * the output space for ungrouping variables. However if an
+             * ungrouping variable is a measure, the field expression value
+             * needed to get the definition needs to be computed in the
+             * input space of the query. There is a test which failed which
+             * caused this code to be here, but this is really a bandaid.
+             *
+             * Some re-work of how to get the definition of a SpaceField
+             * no matter what it is contained in needs to be thought out.
+             *
+             * ... or this check would look at the finalized output of
+             * the namespace and not re-compile ...
+             */
+            let definition = undefined;
+            if (this instanceof QuerySpace) {
+                const qfd = found.getQueryFieldDef(this.qfs());
+                if (qfd &&
+                    typeof qfd != "string" &&
+                    !model.isFilteredAliasedName(qfd)) {
+                    definition = qfd;
+                }
+            }
+            else {
+                definition = found.fieldDef();
+            }
+            if (definition) {
+                head.addReference({
+                    type: found instanceof space_field_1.StructSpaceField
+                        ? "joinReference"
+                        : "fieldReference",
+                    definition,
+                    location: head.location,
+                    text: head.refString,
+                });
+            }
+        }
+        if (rest.length) {
+            if (found instanceof space_field_1.StructSpaceField) {
+                return found.fieldSpace.lookup(rest);
+            }
+            return {
+                error: `'${head}' cannot contain a '${rest[0]}'`,
+                found: undefined,
+            };
+        }
+        return { found, error: undefined };
+    }
+}
+exports.StaticSpace = StaticSpace;
+function isFieldSpace(x) {
+    return x.type == "fieldSpace";
+}
+/**
+ * Based on how things are constructed, the starting field space
+ * can either be another field space or an existing structdef.
+ * Using a SourceSpace allows a class to accept either one
+ * and use either version at some future time.
+ */
+class SourceSpace {
+    constructor(sourceSpec) {
+        this.sourceSpec = sourceSpec;
+        this.spaceSpec = sourceSpec;
+    }
+    get structDef() {
+        if (isFieldSpace(this.spaceSpec)) {
+            if (!this.asStruct) {
+                this.asStruct = this.spaceSpec.structDef();
+            }
+            return this.asStruct;
+        }
+        return this.spaceSpec;
+    }
+    get fieldSpace() {
+        if (isFieldSpace(this.spaceSpec)) {
+            return this.spaceSpec;
+        }
+        if (!this.asFS) {
+            this.asFS = new StaticSpace(this.spaceSpec);
+        }
+        return this.asFS;
+    }
+}
+/**
+ * A FieldSpace which may undergo modification
+ */
+class DynamicSpace extends StaticSpace {
+    constructor(extending) {
+        const source = new SourceSpace(extending);
+        super((0, lodash_1.cloneDeep)(source.structDef));
+        this.completions = [];
+        this.complete = false;
+        this.final = undefined;
+        this.source = source;
+    }
+    /**
+     * Factory for DynamicSpace when there are accept/except edits
+     * @param from A structdef which seeds this space
+     * @param choose A accept/except edit of the "from" fields
+     */
+    static filteredFrom(from, choose) {
+        const edited = new DynamicSpace(from);
+        if (choose) {
+            const names = choose.refs.list;
+            const oldMap = edited.entries();
+            edited.dropEntries();
+            for (const [symbol, value] of oldMap) {
+                const included = !!names.find((f) => f.refString === symbol);
+                const accepting = choose.edit === "accept";
+                if (included === accepting) {
+                    edited.setEntry(symbol, value);
+                }
+            }
+        }
+        return edited;
+    }
+    whenComplete(finalizeStep) {
+        if (this.complete) {
+            finalizeStep();
+        }
+        else {
+            this.completions.push(finalizeStep);
+        }
+    }
+    isComplete() {
+        this.complete = true;
+        for (const step of this.completions) {
+            step();
+        }
+        this.completions = [];
+    }
+    setEntry(name, value) {
+        if (this.final) {
+            throw new Error("Space already final");
+        }
+        super.setEntry(name, value);
+    }
+    addParameters(params) {
+        for (const oneP of params) {
+            this.setEntry(oneP.name, new space_field_1.AbstractParameter(oneP));
+        }
+        return this;
+    }
+    newEntry(name, astEl, entry) {
+        if (this.entry(name)) {
+            astEl.log(`Cannot redefine '${name}'`);
+            return;
+        }
+        this.setEntry(name, entry);
+    }
+    addField(...defs) {
+        for (const def of defs) {
+            // TODO express the "three fields kinds" in a typesafe way
+            // one of three kinds of fields are legal in an explore: expressions ...
+            const elseLog = def.log;
+            const elseType = def.elementType;
+            if (def instanceof ast_1.FieldDeclaration) {
+                const exprField = new space_field_1.ExpressionFieldFromAst(this, def);
+                this.newEntry(exprField.name, def, exprField);
+                // querry (turtle) fields
+            }
+            else if (def instanceof ast_1.TurtleDecl) {
+                const name = def.name;
+                this.newEntry(name, def, new space_field_1.QueryFieldAST(this, def, name));
+            }
+            else if (def instanceof ast_1.RenameField) {
+                if (def.oldName.refString === def.newName) {
+                    def.log("Can't rename field to itself");
+                    continue;
+                }
+                const oldValue = def.oldName.getField(fsPair(this));
+                if (oldValue.found) {
+                    if (oldValue.found instanceof space_field_1.SpaceField) {
+                        this.setEntry(def.newName, new space_field_1.RenameSpaceField(oldValue.found, def.newName, def.location));
+                        this.dropEntry(def.oldName.refString);
+                    }
+                    else {
+                        def.log(`'${def.oldName}' cannot be renamed`);
+                    }
+                }
+                else {
+                    def.log(`Can't rename '${def.oldName}', no such field`);
+                }
+            }
+            else if (def instanceof ast_1.Join) {
+                this.newEntry(def.name.refString, def, new space_field_1.JoinSpaceField(this, def));
+            }
+            else {
+                elseLog(`Internal error: Expected expression, query, or rename, got '${elseType}'`);
+            }
+        }
+    }
+    addFieldDef(fd) {
+        this.setEntry((0, field_utils_1.nameOf)(fd), this.fromFieldDef(fd));
+    }
+    structDef() {
+        const parameters = this.fromStruct.parameters || {};
+        if (this.final === undefined) {
+            this.final = {
+                ...this.fromStruct,
+                fields: [],
+            };
+            // Need to process the entities in specific order
+            const fields = [];
+            const joins = [];
+            const turtles = [];
+            const fixupJoins = [];
+            for (const [name, spaceEntry] of this.entries()) {
+                if (spaceEntry instanceof space_field_1.StructSpaceField) {
+                    joins.push([name, spaceEntry]);
+                }
+                else if (spaceEntry instanceof space_field_1.QueryField) {
+                    turtles.push([name, spaceEntry]);
+                }
+                else if (spaceEntry instanceof space_field_1.SpaceField) {
+                    fields.push([name, spaceEntry]);
+                }
+                else if (spaceEntry instanceof space_field_1.SpaceParam) {
+                    parameters[name] = spaceEntry.parameter();
+                }
+            }
+            const reorderFields = [...fields, ...joins, ...turtles];
+            for (const [fieldName, field] of reorderFields) {
+                if (field instanceof space_field_1.JoinSpaceField) {
+                    const joinStruct = field.join.structDef();
+                    if (!ast_1.ErrorFactory.isErrorStructDef(joinStruct)) {
+                        this.final.fields.push(joinStruct);
+                        fixupJoins.push([field.join, joinStruct]);
+                    }
+                }
+                else {
+                    const fieldDef = field.fieldDef();
+                    if (fieldDef) {
+                        this.final.fields.push(fieldDef);
+                    }
+                    else {
+                        throw new Error(`'${fieldName}' doesn't have a FieldDef`);
+                    }
+                }
+            }
+            if (Object.entries(parameters).length > 0) {
+                this.final.parameters = parameters;
+            }
+            // If we have join expressions, we need to now go back and fill them in
+            for (const [join, missingOn] of fixupJoins) {
+                join.fixupJoinOn(fsPair(this), missingOn);
+            }
+        }
+        this.isComplete();
+        return this.final;
+    }
+}
+exports.DynamicSpace = DynamicSpace;
+/**
+ * A namespace for Query operations. The have an input and an
+ * output set of fields.
+ */
+class QueryOperationSpace extends DynamicSpace {
+    constructor(input) {
+        const inputSpace = new SourceSpace(input);
+        super({ ...inputSpace.structDef, fields: [] });
+        this.queryInput = inputSpace;
+    }
+    inputFS() {
+        return this.queryInput.fieldSpace;
+    }
+    structDef() {
+        // Actually since we have the input structdef and the fields, this feels
+        // like it could return the output structdef of the query op ... except
+        // the pipeline code merges refinements in before asking for the output
+        // struct, and that would require a call back here somehow to get the
+        // pre-refinement fields ... I think possibly this is always a
+        // runtime error
+        throw new Error("StructDef requested from QueryOperationSpace");
+    }
+    addMembers(members) {
+        for (const member of members) {
+            if (member instanceof ast_1.FieldReference) {
+                this.addReference(member);
+            }
+            else if (member instanceof ast_1.WildcardFieldReference) {
+                this.addWild(member);
+            }
+            else {
+                this.addField(member);
+            }
+        }
+    }
+    addReference(ref) {
+        const refName = ref.outputName;
+        if (this.entry(refName)) {
+            ref.log(`Output already has a field named '${refName}'`);
+        }
+        else {
+            this.setEntry(refName, new space_field_1.ReferenceField(ref));
+        }
+    }
+    addWild(wild) {
+        this.setEntry(wild.refString, new space_field_1.WildSpaceField(wild.refString));
+    }
+    log(s) {
+        if (this.astEl) {
+            this.astEl.log(s);
+        }
+    }
+}
+exports.QueryOperationSpace = QueryOperationSpace;
+/**
+ * Reduce and project queries use a "QuerySpace"
+ */
+class QuerySpace extends QueryOperationSpace {
+    constructor() {
+        super(...arguments);
+        this.extendedFields = [];
+    }
+    extendSource(extendField) {
+        this.extendedFields.push(extendField instanceof ast_1.Join
+            ? extendField.name.refString
+            : extendField.defineName);
+        this.extendInto().addField(extendField);
+    }
+    inputFS() {
+        if (this.extendedInput) {
+            return this.extendedInput;
+        }
+        return this.queryInput.fieldSpace;
+    }
+    qfs() {
+        return {
+            in: this.inputFS(),
+            out: this,
+        };
+    }
+    checkUngroup(fn, isExclude) {
+        if (this.lookup([fn]).error) {
+            if (isExclude && this.nestParent) {
+                const parent = this.nestParent;
+                parent.whenComplete(() => {
+                    parent.checkUngroup(fn, isExclude);
+                });
+            }
+            else {
+                const uName = isExclude ? "exclude()" : "all()";
+                fn.log(`${uName} '${fn.refString}' is missing from query output`);
+            }
+        }
+    }
+    extendInto() {
+        if (!this.extendedInput) {
+            this.extendedInput = new DynamicSpace(this.queryInput.sourceSpec);
+        }
+        return this.extendedInput;
+    }
+    getPipeSegment(refineFrom) {
+        const reduceOrProject = this.segType;
+        if (reduceOrProject === "index") {
+            throw new Error("IndexSegment getPipeSegment should not call super");
+        }
+        if (refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.extendSource) {
+            for (const xField of refineFrom.extendSource) {
+                this.extendInto().addFieldDef(xField);
+            }
+        }
+        const segment = {
+            type: reduceOrProject,
+            fields: this.queryFieldDefs(),
+        };
+        segment.fields = (0, field_utils_1.mergeFields)(refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.fields, segment.fields);
+        if (refineFrom === null || refineFrom === void 0 ? void 0 : refineFrom.extendSource) {
+            segment.extendSource = refineFrom.extendSource;
+        }
+        if (this.extendedInput) {
+            const newExtends = [];
+            // have to get the whole structdef because we need all the join
+            // resolution to happen and that only happens at the completion
+            // of the structdef generation.
+            const extendedStruct = this.extendedInput.structDef();
+            for (const extendName of this.extendedFields) {
+                const extendEnt = extendedStruct.fields.find((f) => (0, field_utils_1.nameOf)(f) == extendName);
+                if (extendEnt) {
+                    newExtends.push(extendEnt);
+                }
+            }
+            segment.extendSource = (0, field_utils_1.mergeFields)(segment.extendSource, newExtends);
+        }
+        return segment;
+    }
+    addQueryItems(...qiList) {
+        for (const qi of qiList) {
+            if (qi instanceof ast_1.FieldReference || qi instanceof ast_1.NestReference) {
+                this.addReference(qi);
+            }
+            else if (qi instanceof ast_1.FieldDeclaration) {
+                this.addField(qi);
+            }
+            else if ((0, ast_1.isNestedQuery)(qi)) {
+                const qf = new space_field_1.QueryFieldAST(this.inputFS(), qi, qi.name);
+                qf.nestParent = this;
+                this.setEntry(qi.name, qf);
+            }
+            else {
+                // Compiler will error if we don't handle all cases
+                const _itemNotHandled = qi;
+            }
+        }
+    }
+    canContain(_qd) {
+        return true;
+    }
+    queryFieldDefs() {
+        const fields = [];
+        for (const [name, field] of this.entries()) {
+            if (field instanceof space_field_1.SpaceField) {
+                const fieldQueryDef = field.getQueryFieldDef(this.qfs());
+                if (fieldQueryDef) {
+                    if (this.canContain(fieldQueryDef)) {
+                        fields.push(fieldQueryDef);
+                    }
+                    else {
+                        this.log(`'${name}' not legal in ${this.segType}`);
+                    }
+                }
+                else {
+                    throw new Error(`'${name}' does not have a QueryFieldDef`);
+                }
+            }
+        }
+        this.isComplete();
+        return fields;
+    }
+}
+exports.QuerySpace = QuerySpace;
+class ReduceFieldSpace extends QuerySpace {
+    constructor() {
+        super(...arguments);
+        this.segType = "reduce";
+    }
+}
+exports.ReduceFieldSpace = ReduceFieldSpace;
+class ProjectFieldSpace extends QuerySpace {
+    constructor() {
+        super(...arguments);
+        this.segType = "project";
+    }
+    canContain(qd) {
+        if (typeof qd !== "string") {
+            if (model.isFilteredAliasedName(qd)) {
+                return true;
+            }
+            if (qd.type === "turtle") {
+                this.log("Cannot nest queries in project");
+                return false;
+            }
+            if (qd.aggregate) {
+                this.log("Cannot add aggregate measures to project");
+                return false;
+            }
+        }
+        return true;
+    }
+}
+exports.ProjectFieldSpace = ProjectFieldSpace;
+class IndexFieldSpace extends QueryOperationSpace {
+    constructor() {
+        super(...arguments);
+        this.segType = "index";
+        this.fieldList = new Set();
+    }
+    addReference(ref) {
+        if (ref.getField(this.inputFS()).found) {
+            this.fieldList.add(ref.refString);
+        }
+    }
+    addWild(wild) {
+        this.fieldList.add(wild.refString);
+    }
+    getPipeSegment(refineIndex) {
+        if (refineIndex && refineIndex.fields) {
+            for (const exists of refineIndex.fields) {
+                if (typeof exists == "string") {
+                    this.fieldList.add(exists);
+                }
+            }
+        }
+        this.isComplete();
+        return {
+            type: "index",
+            fields: Array.from(this.fieldList.values()),
+        };
+    }
+}
+exports.IndexFieldSpace = IndexFieldSpace;
+/**
+ * Used to detect references to fields in the statement which defines them
+ */
+class DefSpace {
+    constructor(realFS, circular) {
+        this.realFS = realFS;
+        this.circular = circular;
+        this.type = "fieldSpace";
+        this.foundCircle = false;
+    }
+    structDef() {
+        return this.realFS.structDef();
+    }
+    emptyStructDef() {
+        return this.realFS.emptyStructDef();
+    }
+    lookup(symbol) {
+        if (symbol[0] && symbol[0].refString === this.circular.defineName) {
+            this.foundCircle = true;
+            return {
+                error: `Circular reference to '${this.circular.defineName}' in definition`,
+                found: undefined,
+            };
+        }
+        return this.realFS.lookup(symbol);
+    }
+    getDialect() {
+        return this.realFS.getDialect();
+    }
+    whenComplete(step) {
+        this.realFS.whenComplete(step);
+    }
+}
+exports.DefSpace = DefSpace;
+function fsPair(inFS, outFS) {
+    return {
+        in: inFS,
+        out: outFS || inFS,
+    };
+}
+exports.fsPair = fsPair;
+//# sourceMappingURL=field-space.js.map

package/dist/lang/field-utils.d.ts

@@ -0,0 +1,5 @@
+import { QueryFieldDef, FieldDef } from "../model/malloy_types";
+declare type Field = QueryFieldDef | FieldDef;
+export declare function nameOf(qfd: Field): string;
+export declare function mergeFields<T extends Field>(older: T[] | undefined, newer: T[]): T[];
+export {};

package/dist/lang/field-utils.js

@@ -0,0 +1,33 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mergeFields = exports.nameOf = void 0;
+function nameOf(qfd) {
+    if (typeof qfd == "string") {
+        return qfd;
+    }
+    return qfd.as || qfd.name;
+}
+exports.nameOf = nameOf;
+function mergeFields(older, newer) {
+    if (older == undefined) {
+        return newer;
+    }
+    const redefined = new Set(newer.map((f) => nameOf(f)));
+    const merged = older.filter((f) => !redefined.has(nameOf(f)));
+    merged.push(...newer);
+    return merged;
+}
+exports.mergeFields = mergeFields;
+//# sourceMappingURL=field-utils.js.map

package/dist/lang/index.d.ts

@@ -0,0 +1,9 @@
+export { MalloyTranslator } from "./parse-malloy";
+export type { TranslateResponse, UpdateData, SchemaData, URLData, SQLStructData, } from "./parse-malloy";
+export { exploreQueryWalkerBuilder } from "./parse-tree-walkers/explore-query-walker";
+export type { ExploreClauseRef } from "./parse-tree-walkers/explore-query-walker";
+export { HighlightType } from "./parse-tree-walkers/document-highlight-walker";
+export type { DocumentHighlight } from "./parse-tree-walkers/document-highlight-walker";
+export type { DocumentSymbol } from "./parse-tree-walkers/document-symbol-walker";
+export type { DocumentCompletion } from "./parse-tree-walkers/document-completion-walker";
+export type { LogMessage } from "./parse-log";