@malloydata/malloy 0.0.334 → 0.0.336

package/dist/api/foundation/compile.js

@@ -0,0 +1,391 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Malloy = exports.MalloyError = void 0;
+const lang_1 = require("../../lang");
+const model_1 = require("../../model");
+const sql_block_1 = require("../../model/sql_block");
+const version_1 = require("../../version");
+const readers_1 = require("./readers");
+const document_1 = require("./document");
+const core_1 = require("./core");
+const result_1 = require("./result");
+const MALLOY_INTERNAL_URL = 'internal://internal.malloy';
+// =============================================================================
+// Helper Functions
+// =============================================================================
+function flatDeps(tree) {
+    return [...Object.keys(tree), ...Object.values(tree).map(flatDeps).flat()];
+}
+// =============================================================================
+// MalloyError
+// =============================================================================
+/**
+ * A Malloy error, which may contain log messages produced during compilation.
+ */
+class MalloyError extends Error {
+    /**
+     * An array of log messages produced during compilation.
+     */
+    constructor(message, problems = []) {
+        super(message);
+        this.problems = problems;
+    }
+}
+exports.MalloyError = MalloyError;
+// =============================================================================
+// Malloy Static Class
+// =============================================================================
+class Malloy {
+    static get version() {
+        return version_1.MALLOY_VERSION;
+    }
+    static _parse(source, url, eventStream, options, invalidationKey) {
+        if (url === undefined) {
+            url = new URL(MALLOY_INTERNAL_URL);
+        }
+        let importBaseURL = url;
+        if (options === null || options === void 0 ? void 0 : options.importBaseURL) {
+            importBaseURL = options === null || options === void 0 ? void 0 : options.importBaseURL;
+        }
+        const translator = new lang_1.MalloyTranslator(url.toString(), importBaseURL.toString(), {
+            urls: { [url.toString()]: source },
+        }, eventStream);
+        if (options === null || options === void 0 ? void 0 : options.testEnvironment) {
+            translator.allDialectsEnabled = true;
+        }
+        return new document_1.Parse(translator, invalidationKey);
+    }
+    static parse({ url, urlReader, source, eventStream, options, }) {
+        if (source !== undefined) {
+            return Malloy._parse(source, url, eventStream, options);
+        }
+        else {
+            if (urlReader === undefined) {
+                throw new Error('Internal Error: urlReader is required.');
+            }
+            if (url === undefined) {
+                throw new Error('Internal Error: url is required if source not present.');
+            }
+            return (0, readers_1.readURL)(urlReader, url).then(({ contents, invalidationKey }) => {
+                return Malloy._parse(contents, url, eventStream, options, invalidationKey);
+            });
+        }
+    }
+    /**
+     * Compile a parsed Malloy document.
+     *
+     * @param urlReader Object capable of reading contents of a URL.
+     * @param connections Mapping of connection names to objects capable of reading Malloy schemas.
+     * @param parse The parsed Malloy document.
+     * @param model A compiled model to build upon (optional).
+     * @return A (promise of a) compiled `Model`.
+     */
+    static async compile({ url, source, parse, urlReader, connections, model, refreshSchemaCache, noThrowOnError, eventStream, importBaseURL, cacheManager, }) {
+        var _a, _b, _c, _d, _e;
+        let refreshTimestamp;
+        if (refreshSchemaCache) {
+            refreshTimestamp =
+                typeof refreshSchemaCache === 'number'
+                    ? refreshSchemaCache
+                    : Date.now();
+        }
+        if (url === undefined && source === undefined && parse === undefined) {
+            throw new Error('Internal Error: url, source, or parse required.');
+        }
+        if (url === undefined) {
+            if (parse !== undefined) {
+                url = new URL(parse._translator.sourceURL);
+            }
+            else {
+                url = new URL(MALLOY_INTERNAL_URL);
+            }
+        }
+        const invalidationKeys = {};
+        // Before anything, if we have a URL and not source code, we check if that URL
+        // is cached.
+        if (source === undefined && cacheManager !== undefined) {
+            const cached = await cacheManager.getCachedModelDef(urlReader, url.toString());
+            if (cached) {
+                return new core_1.Model(cached.modelDef, [], // TODO when using a model from cache, should we also store the problems??
+                [url.toString(), ...flatDeps(cached.modelDef.dependencies)]);
+            }
+        }
+        importBaseURL !== null && importBaseURL !== void 0 ? importBaseURL : (importBaseURL = url);
+        let translator;
+        // It's not cached, so we may need to get the actual source
+        const _url = url.toString();
+        if (parse !== undefined) {
+            translator = parse._translator;
+            const invalidationKey = (_a = parse._invalidationKey) !== null && _a !== void 0 ? _a : (await (0, readers_1.getInvalidationKey)(urlReader, url));
+            invalidationKeys[_url] = invalidationKey;
+        }
+        else {
+            if (source === undefined) {
+                const { contents, invalidationKey } = await (0, readers_1.readURL)(urlReader, url);
+                invalidationKeys[_url] = invalidationKey;
+                source = contents;
+            }
+            else {
+                const invalidationKey = await (0, readers_1.getInvalidationKey)(urlReader, url);
+                invalidationKeys[_url] = invalidationKey;
+            }
+            translator = new lang_1.MalloyTranslator(_url, importBaseURL.toString(), {
+                urls: { [_url]: source },
+            }, eventStream);
+        }
+        for (;;) {
+            const result = translator.translate(model === null || model === void 0 ? void 0 : model._modelDef);
+            if (result.final) {
+                if (result.modelDef) {
+                    await (cacheManager === null || cacheManager === void 0 ? void 0 : cacheManager.setCachedModelDef(url.toString(), {
+                        modelDef: result.modelDef,
+                        invalidationKeys,
+                    }));
+                    for (const m of translator.newlyTranslatedDependencies()) {
+                        await (cacheManager === null || cacheManager === void 0 ? void 0 : cacheManager.setCachedModelDef(m.url, {
+                            modelDef: m.modelDef,
+                            invalidationKeys,
+                        }));
+                    }
+                    // If the model wasn't modified, create new Model with result's modelDef
+                    // (which has the queryList) but share the cached QueryModel from input model
+                    const existingQueryModel = !result.modelWasModified && model
+                        ? model.getExistingQueryModel()
+                        : undefined;
+                    return new core_1.Model(result.modelDef, result.problems || [], [...((_b = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _b !== void 0 ? _b : []), ...((_c = result.fromSources) !== null && _c !== void 0 ? _c : [])], existingQueryModel);
+                }
+                else if (noThrowOnError) {
+                    const emptyModel = (0, model_1.mkModelDef)('modelDidNotCompile');
+                    const modelFromCompile = (model === null || model === void 0 ? void 0 : model._modelDef) || emptyModel;
+                    return new core_1.Model(modelFromCompile, result.problems || [], [
+                        ...((_d = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _d !== void 0 ? _d : []),
+                        ...((_e = result.fromSources) !== null && _e !== void 0 ? _e : []),
+                    ]);
+                }
+                else {
+                    const errors = result.problems || [];
+                    const errText = translator.prettyErrors();
+                    throw new MalloyError(`Error(s) compiling model:\n${errText}`, errors);
+                }
+            }
+            else {
+                // Parse incomplete because some external information is required,
+                // there might be more than one of these in a single reply ...
+                if (result.urls) {
+                    for (const neededUrl of result.urls) {
+                        try {
+                            if ((0, readers_1.isInternalURL)(neededUrl)) {
+                                throw new Error('In order to use relative imports, you must compile a file via a URL.');
+                            }
+                            // First, check the cache
+                            if (cacheManager !== undefined) {
+                                const cached = await cacheManager.getCachedModelDef(urlReader, neededUrl);
+                                if (cached) {
+                                    for (const dependency in cached.invalidationKeys) {
+                                        invalidationKeys[dependency] =
+                                            cached.invalidationKeys[dependency];
+                                    }
+                                    translator.update({
+                                        translations: { [neededUrl]: cached.modelDef },
+                                    });
+                                    continue;
+                                }
+                            }
+                            // Otherwise, fetch the URL contents
+                            const { contents, invalidationKey } = await (0, readers_1.readURL)(urlReader, new URL(neededUrl));
+                            const urls = { [neededUrl]: contents };
+                            invalidationKeys[neededUrl] = invalidationKey;
+                            translator.update({ urls });
+                        }
+                        catch (error) {
+                            translator.update({
+                                errors: { urls: { [neededUrl]: error.message } },
+                            });
+                        }
+                    }
+                }
+                const { modelAnnotation } = translator.modelAnnotation(model === null || model === void 0 ? void 0 : model._modelDef);
+                if (result.tables) {
+                    // collect tables by connection name since there may be multiple connections
+                    const tablesByConnection = new Map();
+                    for (const tableKey in result.tables) {
+                        const { connectionName, tablePath } = result.tables[tableKey];
+                        const connectionToTablesMap = tablesByConnection.get(connectionName);
+                        if (connectionToTablesMap === undefined) {
+                            tablesByConnection.set(connectionName, { [tableKey]: tablePath });
+                        }
+                        else {
+                            connectionToTablesMap[tableKey] = tablePath;
+                        }
+                    }
+                    // iterate over connections, fetching schema for all missing tables
+                    for (const [connectionName, tablePathByKey] of tablesByConnection) {
+                        try {
+                            const connection = await connections.lookupConnection(connectionName);
+                            // TODO detect if the union of `Object.keys(tables)` and `Object.keys(errors)` is not the same
+                            //      as `Object.keys(tablePathByKey)`, i.e. that all tables are accounted for. Otherwise
+                            //      the translator runs into an infinite loop fetching tables.
+                            const { schemas: tables, errors } = await Malloy.safelyFetchTableSchema(connection, tablePathByKey, {
+                                refreshTimestamp,
+                                modelAnnotation,
+                            });
+                            translator.update({ tables, errors: { tables: errors } });
+                        }
+                        catch (error) {
+                            // There was an exception getting the connection, associate that error
+                            // with all its tables
+                            const tables = {};
+                            const errors = {};
+                            for (const tableKey in tablePathByKey) {
+                                errors[tableKey] = error.toString();
+                            }
+                            translator.update({ tables, errors: { tables: errors } });
+                        }
+                    }
+                }
+                if (result.compileSQL) {
+                    // Unlike other requests, these do not come in batches
+                    const toCompile = result.compileSQL;
+                    const connectionName = toCompile.connection;
+                    const key = (0, sql_block_1.sqlKey)(toCompile.connection, toCompile.selectStr);
+                    try {
+                        const conn = await connections.lookupConnection(connectionName);
+                        const resolved = await conn.fetchSchemaForSQLStruct(toCompile, {
+                            refreshTimestamp,
+                            modelAnnotation,
+                        });
+                        if (resolved.error) {
+                            translator.update({
+                                errors: {
+                                    compileSQL: {
+                                        [key]: resolved.error,
+                                    },
+                                },
+                            });
+                        }
+                        if (resolved.structDef) {
+                            translator.update({
+                                compileSQL: { [key]: resolved.structDef },
+                            });
+                        }
+                    }
+                    catch (error) {
+                        const errors = {};
+                        errors[key] = error.toString();
+                        translator.update({ errors: { compileSQL: errors } });
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * A dialect must provide a response for every table, or the translator loop
+     * will never exit. Because there was a time when this happened, we throw
+     * instead of looping forever, but the fix is to correct the dialect.
+     */
+    static async safelyFetchTableSchema(connection, toFetch, opts) {
+        const ret = await connection.fetchSchemaForTables(toFetch, opts);
+        for (const req of Object.keys(toFetch)) {
+            if (ret.schemas[req] === undefined && ret.errors[req] === undefined) {
+                throw new Error(`Schema fetch error for ${connection.name}, no response for ${req} from ${connection.dialectName}`);
+            }
+        }
+        return ret;
+    }
+    static async run({ connections, preparedResult, sqlStruct, connection, options, }) {
+        if (!connection) {
+            if (!connections) {
+                throw new Error('Internal Error: Connection or LookupConnection<Connection> must be provided.');
+            }
+            const connectionName = (sqlStruct === null || sqlStruct === void 0 ? void 0 : sqlStruct.connection) || (preparedResult === null || preparedResult === void 0 ? void 0 : preparedResult.connectionName);
+            connection = await connections.lookupConnection(connectionName);
+        }
+        if (sqlStruct) {
+            const data = await connection.runSQL(sqlStruct.selectStr);
+            return new result_1.Result({
+                structs: [sqlStruct],
+                sql: sqlStruct.selectStr,
+                result: data.rows,
+                totalRows: data.totalRows,
+                runStats: data.runStats,
+                lastStageName: sqlStruct.name,
+                // TODO feature-sql-block There is no malloy code...
+                malloy: '',
+                connectionName: sqlStruct.connection,
+                // TODO feature-sql-block There is no source explore...
+                sourceExplore: '',
+                sourceFilters: [],
+                profilingUrl: data.profilingUrl,
+            }, (0, model_1.mkModelDef)('empty_model'));
+        }
+        else if (preparedResult) {
+            const result = await connection.runSQL(preparedResult.sql, options);
+            return new result_1.Result({
+                ...preparedResult._rawQuery,
+                result: result.rows,
+                totalRows: result.totalRows,
+                runStats: result.runStats,
+                profilingUrl: result.profilingUrl,
+            }, preparedResult._modelDef);
+        }
+        else {
+            throw new Error('Internal error: sqlStruct or preparedResult must be provided.');
+        }
+    }
+    static async *runStream({ connections, preparedResult, sqlStruct, connection, options, }) {
+        if (sqlStruct === undefined && preparedResult === undefined) {
+            throw new Error('Internal error: sqlBlock or preparedResult must be provided.');
+        }
+        const connectionName = (sqlStruct === null || sqlStruct === void 0 ? void 0 : sqlStruct.connection) || (preparedResult === null || preparedResult === void 0 ? void 0 : preparedResult.connectionName);
+        if (connection === undefined) {
+            if (connections === undefined) {
+                throw new Error('Internal Error: Connection or LookupConnection<Connection> must be provided.');
+            }
+            connection = await connections.lookupConnection(connectionName);
+        }
+        // TODO is there a better way to handle this case? Just require StreamingConnections?
+        if (!connection.canStream()) {
+            throw new Error(`Connection '${connectionName}' cannot stream results.`);
+        }
+        let sql;
+        let resultExplore;
+        if (sqlStruct) {
+            resultExplore = new core_1.Explore(sqlStruct);
+            sql = sqlStruct.selectStr;
+        }
+        else if (preparedResult !== undefined) {
+            resultExplore = preparedResult.resultExplore;
+            sql = preparedResult.sql;
+        }
+        else {
+            throw new Error('Internal error: sqlStruct or preparedResult must be provided.');
+        }
+        let index = 0;
+        for await (const row of connection.runSQLStream(sql, options)) {
+            yield new result_1.DataRecord(row, index, resultExplore, undefined, undefined);
+            index += 1;
+        }
+    }
+    static async estimateQueryCost({ connections, preparedResult, sqlStruct, }) {
+        if (!connections) {
+            throw new Error('Internal Error: Connection or LookupConnection<Connection> must be provided.');
+        }
+        const connectionName = (sqlStruct === null || sqlStruct === void 0 ? void 0 : sqlStruct.connection) || (preparedResult === null || preparedResult === void 0 ? void 0 : preparedResult.connectionName);
+        const connection = await connections.lookupConnection(connectionName);
+        if (sqlStruct) {
+            return await connection.estimateQueryCost(sqlStruct.selectStr);
+        }
+        else if (preparedResult) {
+            return await connection.estimateQueryCost(preparedResult.sql);
+        }
+        else {
+            throw new Error('Internal error: sqlStruct or preparedResult must be provided.');
+        }
+    }
+}
+exports.Malloy = Malloy;
+//# sourceMappingURL=compile.js.map