@malloydata/malloy 0.0.232-dev250118020814 → 0.0.232-dev250127221938

package/dist/index.d.ts

@@ -4,10 +4,10 @@ export type { QueryDataRow, StructDef, TableSourceDef, SQLSourceDef, SourceDef,
 export { isSourceDef, Segment, isLeafAtomic, isJoined, isJoinedSource, isSamplingEnable, isSamplingPercent, isSamplingRows, isRepeatedRecord, isScalarArray, mkArrayDef, mkFieldDef, expressionIsAggregate, expressionIsAnalytic, expressionIsCalculation, expressionIsScalar, expressionIsUngroupedAggregate, indent, composeSQLExpr, } from './model';
 export { MalloyTranslator, } from './lang';
 export type { LogMessage, TranslateResponse } from './lang';
-export { Model, Malloy, Runtime, AtomicFieldType, ConnectionRuntime, SingleConnectionRuntime, EmptyURLReader, InMemoryURLReader, FixedConnectionMap, MalloyError, JoinRelationship, SourceRelationship, DateTimeframe, TimestampTimeframe, PreparedResult, Result, QueryMaterializer, CSVWriter, JSONWriter, Parse, DataWriter, Explore, } from './malloy';
-export type { PreparedQuery, Field, AtomicField, ExploreField, QueryField, SortableField, DataArray, DataRecord, DataColumn, DataArrayOrRecord, Loggable, ModelMaterializer, DocumentTablePath, DocumentSymbol, ResultJSON, PreparedResultMaterializer, ExploreMaterializer, WriteStream, SerializedExplore, DateField, TimestampField, } from './malloy';
+export { Model, Malloy, Runtime, AtomicFieldType, ConnectionRuntime, SingleConnectionRuntime, EmptyURLReader, InMemoryURLReader, FixedConnectionMap, MalloyError, JoinRelationship, SourceRelationship, DateTimeframe, TimestampTimeframe, PreparedResult, Result, QueryMaterializer, CSVWriter, JSONWriter, Parse, DataWriter, Explore, InMemoryModelCache, CacheManager, } from './malloy';
+export type { PreparedQuery, Field, AtomicField, ExploreField, QueryField, SortableField, DataArray, DataRecord, DataColumn, DataArrayOrRecord, Loggable, ModelMaterializer, DocumentTablePath, DocumentSymbol, ResultJSON, PreparedResultMaterializer, ExploreMaterializer, WriteStream, SerializedExplore, ModelCache, CachedModel, DateField, TimestampField, } from './malloy';
 export type { QueryOptionsReader, RunSQLOptions } from './run_sql_options';
-export type { EventStream, ModelString, ModelURL, QueryString, QueryURL, URLReader, } from './runtime_types';
+export type { EventStream, ModelString, ModelURL, QueryString, QueryURL, URLReader, InvalidationKey, } from './runtime_types';
 export type { Connection, ConnectionConfig, ConnectionFactory, ConnectionParameter, ConnectionParameterValue, ConnectionConfigSchema, FetchSchemaOptions, InfoConnection, LookupConnection, PersistSQLResults, PooledConnection, TestableConnection, StreamingConnection, } from './connection/types';
 export { toAsyncGenerator } from './connection_utils';
 export { type TagParse, Tag, type TagDict } from './tags';

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryURLReader = exports.EmptyURLReader = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.AtomicFieldType = exports.Runtime = exports.Malloy = exports.Model = exports.MalloyTranslator = exports.composeSQLExpr = exports.indent = exports.expressionIsUngroupedAggregate = exports.expressionIsScalar = exports.expressionIsCalculation = exports.expressionIsAnalytic = exports.expressionIsAggregate = exports.mkFieldDef = exports.mkArrayDef = exports.isScalarArray = exports.isRepeatedRecord = exports.isSamplingRows = exports.isSamplingPercent = exports.isSamplingEnable = exports.isJoinedSource = exports.isJoined = exports.isLeafAtomic = exports.Segment = exports.isSourceDef = exports.TinyParser = exports.Dialect = exports.spread = exports.literal = exports.variadicParam = exports.param = exports.makeParam = exports.sql = exports.maxScalar = exports.minAggregate = exports.anyExprType = exports.minScalar = exports.overload = exports.qtz = exports.arg = exports.registerDialect = exports.MySQLDialect = exports.SnowflakeDialect = exports.PostgresDialect = exports.TrinoDialect = exports.StandardSQLDialect = exports.DuckDBDialect = void 0;
-exports.Tag = exports.toAsyncGenerator = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = void 0;
+exports.Tag = exports.toAsyncGenerator = exports.CacheManager = exports.InMemoryModelCache = exports.Explore = exports.DataWriter = exports.Parse = exports.JSONWriter = exports.CSVWriter = exports.QueryMaterializer = exports.Result = exports.PreparedResult = exports.TimestampTimeframe = exports.DateTimeframe = exports.SourceRelationship = exports.JoinRelationship = exports.MalloyError = exports.FixedConnectionMap = void 0;
 /*
  * Copyright 2023 Google LLC
  *
@@ -94,6 +94,8 @@ Object.defineProperty(exports, "JSONWriter", { enumerable: true, get: function (
 Object.defineProperty(exports, "Parse", { enumerable: true, get: function () { return malloy_1.Parse; } });
 Object.defineProperty(exports, "DataWriter", { enumerable: true, get: function () { return malloy_1.DataWriter; } });
 Object.defineProperty(exports, "Explore", { enumerable: true, get: function () { return malloy_1.Explore; } });
+Object.defineProperty(exports, "InMemoryModelCache", { enumerable: true, get: function () { return malloy_1.InMemoryModelCache; } });
+Object.defineProperty(exports, "CacheManager", { enumerable: true, get: function () { return malloy_1.CacheManager; } });
 var connection_utils_1 = require("./connection_utils");
 Object.defineProperty(exports, "toAsyncGenerator", { enumerable: true, get: function () { return connection_utils_1.toAsyncGenerator; } });
 var tags_1 = require("./tags");

package/dist/malloy.d.ts

@@ -3,7 +3,7 @@ import { RunSQLOptions } from './run_sql_options';
 import { DocumentCompletion as DocumentCompletionDefinition, DocumentSymbol as DocumentSymbolDefinition, LogMessage, MalloyTranslator } from './lang';
 import { DocumentHelpContext } from './lang/parse-tree-walkers/document-help-context-walker';
 import { CompiledQuery, DocumentLocation, DocumentReference, BooleanFieldDef, JSONFieldDef, NumberFieldDef, StringFieldDef, FilterCondition, Query as InternalQuery, ModelDef, DocumentPosition as ModelDocumentPosition, NamedQuery, QueryData, QueryDataRow, QueryResult, SearchIndexResult, SearchValueMapResult, StructDef, TurtleDef, NativeUnsupportedFieldDef, QueryRunStats, ImportLocation, Annotation, SQLSentence, SQLSourceDef, AtomicFieldDef, DateFieldDef, TimestampFieldDef, SourceDef, QueryToMaterialize } from './model';
-import { EventStream, ModelString, ModelURL, QueryString, QueryURL, URLReader } from './runtime_types';
+import { EventStream, InvalidationKey, ModelString, ModelURL, QueryString, QueryURL, URLReader } from './runtime_types';
 import { Connection, FetchSchemaOptions, InfoConnection, LookupConnection } from './connection/types';
 import { Tag, TagParse, TagParseSpec, Taggable } from './tags';
 import { Dialect } from './dialect';
@@ -30,6 +30,19 @@ interface CompileQueryOptions {
     eventStream?: EventStream;
     defaultRowLimit?: number;
 }
+type Compilable = {
+    parse: Parse;
+    url?: undefined;
+    source?: undefined;
+} | {
+    url: URL;
+    parse?: undefined;
+    source?: undefined;
+} | {
+    source: string;
+    parse?: undefined;
+    url?: undefined;
+};
 export declare class Malloy {
     static get version(): string;
     private static _parse;
@@ -68,13 +81,13 @@ export declare class Malloy {
      * @param model A compiled model to build upon (optional).
      * @return A (promise of a) compiled `Model`.
      */
-    static compile({ urlReader, connections, parse, model, refreshSchemaCache, noThrowOnError, eventStream, replaceMaterializedReferences, materializedTablePrefix, }: {
+    static compile({ url, source, parse, urlReader, connections, model, refreshSchemaCache, noThrowOnError, eventStream, replaceMaterializedReferences, materializedTablePrefix, importBaseURL, cacheManager, }: {
         urlReader: URLReader;
         connections: LookupConnection<InfoConnection>;
-        parse: Parse;
         model?: Model;
         replaceMaterializedReferences?: boolean;
-    } & CompileOptions & CompileQueryOptions): Promise<Model>;
+        cacheManager?: CacheManager;
+    } & Compilable & CompileOptions & CompileQueryOptions & ParseOptions): Promise<Model>;
     /**
      * A dialect must provide a response for every table, or the translator loop
      * will never exit. Because there was a time when this happened, we throw
@@ -273,7 +286,8 @@ export declare class PreparedQuery implements Taggable {
  */
 export declare class Parse {
     private translator;
-    constructor(translator: MalloyTranslator);
+    private invalidationKey?;
+    constructor(translator: MalloyTranslator, invalidationKey?: InvalidationKey | undefined);
     /**
      * Retrieve the symbols defined in the parsed document.
      *
@@ -292,6 +306,7 @@ export declare class Parse {
      */
     get tablePathInfo(): DocumentTablePath[];
     get _translator(): MalloyTranslator;
+    get _invalidationKey(): InvalidationKey | undefined;
     completions(position: {
         line: number;
         character: number;
@@ -473,15 +488,24 @@ export declare class PreparedResult implements Taggable {
  * Useful for scenarios in which `import` statements are not required.
  */
 export declare class EmptyURLReader implements URLReader {
-    readURL(_url: URL): Promise<string>;
+    readURL(_url: URL): Promise<{
+        contents: string;
+        invalidationKey: InvalidationKey;
+    }>;
+    getInvalidationKey(_url: URL): Promise<InvalidationKey>;
 }
 /**
  * A URL reader backed by an in-memory mapping of URL contents.
  */
 export declare class InMemoryURLReader implements URLReader {
-    private files;
+    protected files: Map<string, string>;
     constructor(files: Map<string, string>);
-    readURL(url: URL): Promise<string>;
+    readURL(url: URL): Promise<{
+        contents: string;
+        invalidationKey: InvalidationKey;
+    }>;
+    getInvalidationKey(url: URL): Promise<InvalidationKey>;
+    private invalidationKey;
 }
 /**
  * A fixed mapping of connection names to connections.
@@ -726,6 +750,13 @@ export declare class ExploreField extends Explore {
     get parentExplore(): Explore;
     get sourceClasses(): string[];
 }
+type Connectionable = {
+    connection: Connection;
+    connections?: undefined;
+} | {
+    connections: LookupConnection<Connection>;
+    connection?: undefined;
+};
 /**
  * An environment for compiling and running Malloy queries.
  */
@@ -734,11 +765,16 @@ export declare class Runtime {
     private _urlReader;
     private _connections;
     private _eventStream;
-    constructor(runtime: LookupConnection<Connection> & URLReader);
-    constructor(urls: URLReader, connections: LookupConnection<Connection>, eventStream?: EventStream);
-    constructor(urls: URLReader, connection: Connection, eventStream?: EventStream);
-    constructor(connection: Connection, eventStream?: EventStream);
-    constructor(connections: LookupConnection<Connection>, eventStream?: EventStream);
+    private _cacheManager;
+    constructor({ urlReader, connections, connection, eventStream, cacheManager, }: {
+        urlReader?: URLReader;
+        eventStream?: EventStream;
+        cacheManager?: CacheManager;
+    } & Connectionable);
+    /**
+     * @return The `CacheManager` for this runtime instance.
+     */
+    get cacheManager(): CacheManager | undefined;
     /**
      * @return The `URLReader` for this runtime instance.
      */
@@ -823,15 +859,19 @@ export declare class Runtime {
 }
 export declare class ConnectionRuntime extends Runtime {
     readonly rawConnections: Connection[];
-    constructor(urls: URLReader, connections: Connection[]);
-    constructor(connections: Connection[]);
+    constructor({ urlReader, connections, }: {
+        urlReader?: URLReader;
+        connections: Connection[];
+    });
 }
 export declare class SingleConnectionRuntime<T extends Connection = Connection> extends Runtime {
     readonly connection: T;
-    constructor(urlReader: URLReader, connection: T);
-    constructor(connection: T);
-    constructor(connection: T, eventStream: EventStream);
-    constructor(urlReader: URLReader, connection: T, eventStream: EventStream);
+    constructor({ urlReader, connection, eventStream, cacheManager, }: {
+        urlReader?: URLReader;
+        eventStream?: EventStream;
+        cacheManager?: CacheManager;
+        connection: T;
+    });
     get supportsNesting(): boolean;
     quote(column: string): string;
     get dialect(): Dialect;
@@ -1257,4 +1297,33 @@ export declare class CSVWriter extends DataWriter {
     private getRowMatrix;
     process(data: AsyncIterableIterator<DataRecord>): Promise<void>;
 }
+interface CacheGetModelDefResponse {
+    modelDef: ModelDef;
+    invalidationKeys: {
+        [url: string]: InvalidationKey;
+    };
+}
+export interface ModelCache {
+    getModel(url: URL): Promise<CachedModel | undefined>;
+    setModel(url: URL, cachedModel: CachedModel): Promise<boolean>;
+}
+export declare class CacheManager {
+    private modelCache;
+    private modelDependencies;
+    private modelInvalidationKeys;
+    constructor(modelCache: ModelCache);
+    getCachedModelDef(urlReader: URLReader, url: string): Promise<CacheGetModelDefResponse | undefined>;
+    setCachedModelDef(url: string, cachedModel: CachedModel): Promise<boolean>;
+}
+export interface CachedModel {
+    modelDef: ModelDef;
+    invalidationKeys: {
+        [url: string]: InvalidationKey;
+    };
+}
+export declare class InMemoryModelCache implements ModelCache {
+    private readonly models;
+    getModel(url: URL): Promise<CachedModel | undefined>;
+    setModel(url: URL, cachedModel: CachedModel): Promise<boolean>;
+}
 export {};

package/dist/malloy.js

@@ -22,20 +22,22 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CSVWriter = exports.JSONWriter = exports.DataWriter = exports.DataRecord = exports.DataArray = exports.Result = exports.ExploreMaterializer = exports.PreparedResultMaterializer = exports.QueryMaterializer = exports.ModelMaterializer = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.Runtime = exports.ExploreField = exports.JoinRelationship = exports.QueryField = exports.Query = exports.StringField = exports.UnsupportedField = exports.JSONField = exports.BooleanField = exports.NumberField = exports.TimestampField = exports.DateField = exports.TimestampTimeframe = exports.DateTimeframe = exports.AtomicField = exports.AtomicFieldType = exports.Explore = exports.SourceRelationship = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.PreparedResult = exports.DocumentCompletion = exports.DocumentSymbol = exports.DocumentPosition = exports.DocumentRange = exports.DocumentTablePath = exports.Parse = exports.PreparedQuery = exports.Model = exports.MalloyError = exports.Malloy = void 0;
+exports.InMemoryModelCache = exports.CacheManager = exports.CSVWriter = exports.JSONWriter = exports.DataWriter = exports.DataRecord = exports.DataArray = exports.Result = exports.ExploreMaterializer = exports.PreparedResultMaterializer = exports.QueryMaterializer = exports.ModelMaterializer = exports.SingleConnectionRuntime = exports.ConnectionRuntime = exports.Runtime = exports.ExploreField = exports.JoinRelationship = exports.QueryField = exports.Query = exports.StringField = exports.UnsupportedField = exports.JSONField = exports.BooleanField = exports.NumberField = exports.TimestampField = exports.DateField = exports.TimestampTimeframe = exports.DateTimeframe = exports.AtomicField = exports.AtomicFieldType = exports.Explore = exports.SourceRelationship = exports.FixedConnectionMap = exports.InMemoryURLReader = exports.EmptyURLReader = exports.PreparedResult = exports.DocumentCompletion = exports.DocumentSymbol = exports.DocumentPosition = exports.DocumentRange = exports.DocumentTablePath = exports.Parse = exports.PreparedQuery = exports.Model = exports.MalloyError = exports.Malloy = void 0;
 const lang_1 = require("./lang");
 const model_1 = require("./model");
 const luxon_1 = require("luxon");
 const tags_1 = require("./tags");
 const dialect_1 = require("./dialect");
 const version_1 = require("./version");
+const uuid_1 = require("uuid");
+const MALLOY_INTERNAL_URL = 'internal://internal.malloy';
 class Malloy {
     static get version() {
         return version_1.MALLOY_VERSION;
     }
-    static _parse(source, url, eventStream, options) {
+    static _parse(source, url, eventStream, options, invalidationKey) {
         if (url === undefined) {
-            url = new URL('internal://internal.malloy');
+            url = new URL(MALLOY_INTERNAL_URL);
         }
         let importBaseURL = url;
         if (options === null || options === void 0 ? void 0 : options.importBaseURL) {
@@ -47,7 +49,7 @@ class Malloy {
         if (options === null || options === void 0 ? void 0 : options.testEnvironment) {
             translator.allDialectsEnabled = true;
         }
-        return new Parse(translator);
+        return new Parse(translator, invalidationKey);
     }
     static parse({ url, urlReader, source, eventStream, options, }) {
         if (source !== undefined) {
@@ -60,8 +62,8 @@ class Malloy {
             if (url === undefined) {
                 throw new Error('Internal Error: url is required if source not present.');
             }
-            return urlReader.readURL(url).then(source => {
-                return Malloy._parse(source, url, eventStream, options);
+            return readURL(urlReader, url).then(({ contents, invalidationKey }) => {
+                return Malloy._parse(contents, url, eventStream, options, invalidationKey);
             });
         }
     }
@@ -74,8 +76,8 @@ class Malloy {
      * @param model A compiled model to build upon (optional).
      * @return A (promise of a) compiled `Model`.
      */
-    static async compile({ urlReader, connections, parse, model, refreshSchemaCache, noThrowOnError, eventStream, replaceMaterializedReferences, materializedTablePrefix, }) {
-        var _a, _b, _c, _d;
+    static async compile({ url, source, parse, urlReader, connections, model, refreshSchemaCache, noThrowOnError, eventStream, replaceMaterializedReferences, materializedTablePrefix, importBaseURL, cacheManager, }) {
+        var _a, _b, _c, _d, _e;
         let refreshTimestamp;
         if (refreshSchemaCache) {
             refreshTimestamp =
@@ -83,12 +85,67 @@ class Malloy {
                     ? refreshSchemaCache
                     : Date.now();
         }
-        const translator = parse._translator;
+        if (url === undefined && source === undefined && parse === undefined) {
+            throw new Error('Internal Error: url, source, or parse required.');
+        }
+        if (url === undefined) {
+            if (parse !== undefined) {
+                url = new URL(parse._translator.sourceURL);
+            }
+            else {
+                url = new URL(MALLOY_INTERNAL_URL);
+            }
+        }
+        const invalidationKeys = {};
+        // Before anything, if we have a URL and not source code, we check if that URL
+        // is cached.
+        if (source === undefined && cacheManager !== undefined) {
+            const cached = await cacheManager.getCachedModelDef(urlReader, url.toString());
+            if (cached) {
+                return new Model(cached.modelDef, [], // TODO when using a model from cache, should we also store the problems??
+                [url.toString(), ...flatDeps(cached.modelDef.dependencies)]
+                // TODO maybe implement referenceAt and importAt by re-translating the model?
+                );
+            }
+        }
+        importBaseURL !== null && importBaseURL !== void 0 ? importBaseURL : (importBaseURL = url);
+        let translator;
+        // It's not cached, so we may need to get the actual source
+        const _url = url.toString();
+        if (parse !== undefined) {
+            translator = parse._translator;
+            const invalidationKey = (_a = parse._invalidationKey) !== null && _a !== void 0 ? _a : (await getInvalidationKey(urlReader, url));
+            invalidationKeys[_url] = invalidationKey;
+        }
+        else {
+            if (source === undefined) {
+                const { contents, invalidationKey } = await readURL(urlReader, url);
+                invalidationKeys[_url] = invalidationKey;
+                source = contents;
+            }
+            else {
+                const invalidationKey = await getInvalidationKey(urlReader, url);
+                invalidationKeys[_url] = invalidationKey;
+            }
+            translator = new lang_1.MalloyTranslator(_url, importBaseURL.toString(), {
+                urls: { [_url]: source },
+            }, eventStream);
+        }
         for (;;) {
             const result = translator.translate(model === null || model === void 0 ? void 0 : model._modelDef);
             if (result.final) {
                 if (result.modelDef) {
-                    return new Model(result.modelDef, result.problems || [], [...((_a = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _a !== void 0 ? _a : []), ...((_b = result.fromSources) !== null && _b !== void 0 ? _b : [])], (position) => translator.referenceAt(position), (position) => translator.importAt(position));
+                    await (cacheManager === null || cacheManager === void 0 ? void 0 : cacheManager.setCachedModelDef(url.toString(), {
+                        modelDef: result.modelDef,
+                        invalidationKeys,
+                    }));
+                    for (const model of translator.newlyTranslatedDependencies()) {
+                        await (cacheManager === null || cacheManager === void 0 ? void 0 : cacheManager.setCachedModelDef(model.url, {
+                            modelDef: model.modelDef,
+                            invalidationKeys,
+                        }));
+                    }
+                    return new Model(result.modelDef, result.problems || [], [...((_b = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _b !== void 0 ? _b : []), ...((_c = result.fromSources) !== null && _c !== void 0 ? _c : [])], (position) => translator.referenceAt(position), (position) => translator.importAt(position));
                 }
                 else if (noThrowOnError) {
                     const emptyModel = {
@@ -99,7 +156,7 @@ class Malloy {
                         queryList: [],
                     };
                     const modelFromCompile = (model === null || model === void 0 ? void 0 : model._modelDef) || emptyModel;
-                    return new Model(modelFromCompile, result.problems || [], [...((_c = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _c !== void 0 ? _c : []), ...((_d = result.fromSources) !== null && _d !== void 0 ? _d : [])], (position) => translator.referenceAt(position), (position) => translator.importAt(position));
+                    return new Model(modelFromCompile, result.problems || [], [...((_d = model === null || model === void 0 ? void 0 : model.fromSources) !== null && _d !== void 0 ? _d : []), ...((_e = result.fromSources) !== null && _e !== void 0 ? _e : [])], (position) => translator.referenceAt(position), (position) => translator.importAt(position));
                 }
                 else {
                     const errors = result.problems || [];
@@ -113,11 +170,27 @@ class Malloy {
                 if (result.urls) {
                     for (const neededUrl of result.urls) {
                         try {
-                            if (neededUrl.startsWith('internal://')) {
+                            if (isInternalURL(neededUrl)) {
                                 throw new Error('In order to use relative imports, you must compile a file via a URL.');
                             }
-                            const neededText = await urlReader.readURL(new URL(neededUrl));
-                            const urls = { [neededUrl]: neededText };
+                            // First, check the cache
+                            if (cacheManager !== undefined) {
+                                const cached = await cacheManager.getCachedModelDef(urlReader, neededUrl);
+                                if (cached) {
+                                    for (const dependency in cached.invalidationKeys) {
+                                        invalidationKeys[dependency] =
+                                            cached.invalidationKeys[dependency];
+                                    }
+                                    translator.update({
+                                        translations: { [neededUrl]: cached.modelDef },
+                                    });
+                                    continue;
+                                }
+                            }
+                            // Otherwise, fetch the URL contents
+                            const { contents, invalidationKey } = await readURL(urlReader, new URL(neededUrl));
+                            const urls = { [neededUrl]: contents };
+                            invalidationKeys[neededUrl] = invalidationKey;
                             translator.update({ urls });
                         }
                         catch (error) {
@@ -548,8 +621,9 @@ exports.PreparedQuery = PreparedQuery;
  * A parsed Malloy document.
  */
 class Parse {
-    constructor(translator) {
+    constructor(translator, invalidationKey) {
         this.translator = translator;
+        this.invalidationKey = invalidationKey;
     }
     /**
      * Retrieve the symbols defined in the parsed document.
@@ -577,6 +651,9 @@ class Parse {
     get _translator() {
         return this.translator;
     }
+    get _invalidationKey() {
+        return this.invalidationKey;
+    }
     completions(position) {
         return (this.translator.completions(position).completions || []).map(completion => new DocumentCompletion(completion));
     }
@@ -846,6 +923,9 @@ class EmptyURLReader {
     async readURL(_url) {
         throw new Error('No files.');
     }
+    async getInvalidationKey(_url) {
+        throw new Error('No files.');
+    }
 }
 exports.EmptyURLReader = EmptyURLReader;
 /**
@@ -858,12 +938,30 @@ class InMemoryURLReader {
     async readURL(url) {
         const file = this.files.get(url.toString());
         if (file !== undefined) {
-            return Promise.resolve(file);
+            return Promise.resolve({
+                contents: file,
+                invalidationKey: this.invalidationKey(url, file),
+            });
+        }
+        else {
+            throw new Error(`File not found '${url}'`);
+        }
+    }
+    async getInvalidationKey(url) {
+        const file = this.files.get(url.toString());
+        if (file !== undefined) {
+            return Promise.resolve(this.invalidationKey(url, file));
         }
         else {
             throw new Error(`File not found '${url}'`);
         }
     }
+    invalidationKey(url, contents) {
+        if (isInternalURL(url.toString())) {
+            return null;
+        }
+        return hashForInvalidationKey(contents);
+    }
 }
 exports.InMemoryURLReader = InMemoryURLReader;
 /**
@@ -1560,39 +1658,29 @@ exports.ExploreField = ExploreField;
  * An environment for compiling and running Malloy queries.
  */
 class Runtime {
-    constructor(...args) {
+    constructor({ urlReader, connections, connection, eventStream, cacheManager, }) {
         this.isTestRuntime = false;
-        let urlReader;
-        let connections;
-        let eventStream;
-        for (const arg of args) {
-            if (arg === undefined) {
-                continue;
-            }
-            else if (isURLReader(arg)) {
-                urlReader = arg;
-            }
-            else if (isLookupConnection(arg)) {
-                connections = arg;
-            }
-            else if (isEventStream(arg)) {
-                eventStream = arg;
-            }
-            else {
-                connections = {
-                    lookupConnection: () => Promise.resolve(arg),
-                };
+        if (connections === undefined) {
+            if (connection === undefined) {
+                throw new Error('A LookupConnection<Connection> or Connection is required.');
             }
+            connections = {
+                lookupConnection: () => Promise.resolve(connection),
+            };
         }
         if (urlReader === undefined) {
             urlReader = new EmptyURLReader();
         }
-        if (connections === undefined) {
-            throw new Error('A LookupConnection<Connection> or Connection is required.');
-        }
         this._urlReader = urlReader;
         this._connections = connections;
         this._eventStream = eventStream;
+        this._cacheManager = cacheManager;
+    }
+    /**
+     * @return The `CacheManager` for this runtime instance.
+     */
+    get cacheManager() {
+        return this._cacheManager;
     }
     /**
      * @return The `URLReader` for this runtime instance.
@@ -1629,28 +1717,20 @@ class Runtime {
                 options = { ...options, testEnvironment: true };
             }
         }
+        const compilable = source instanceof URL ? { url: source } : { source };
         return new ModelMaterializer(this, async () => {
-            const parse = source instanceof URL
-                ? await Malloy.parse({
-                    url: source,
-                    urlReader: this.urlReader,
-                    eventStream: this.eventStream,
-                    options,
-                })
-                : Malloy.parse({
-                    source,
-                    eventStream: this.eventStream,
-                    options,
-                });
             return Malloy.compile({
+                ...compilable,
                 urlReader: this.urlReader,
                 connections: this.connections,
-                parse,
                 refreshSchemaCache,
                 noThrowOnError,
                 eventStream: this.eventStream,
                 replaceMaterializedReferences: options === null || options === void 0 ? void 0 : options.replaceMaterializedReferences,
                 materializedTablePrefix: options === null || options === void 0 ? void 0 : options.materializedTablePrefix,
+                importBaseURL: options === null || options === void 0 ? void 0 : options.importBaseURL,
+                testEnvironment: options === null || options === void 0 ? void 0 : options.testEnvironment,
+                cacheManager: this.cacheManager,
             });
         }, options);
     }
@@ -1741,40 +1821,23 @@ class Runtime {
 }
 exports.Runtime = Runtime;
 class ConnectionRuntime extends Runtime {
-    constructor(urlsOrConnections, maybeConnections) {
-        if (maybeConnections === undefined) {
-            const connections = urlsOrConnections;
-            super(FixedConnectionMap.fromArray(connections));
-            this.rawConnections = connections;
-        }
-        else {
-            const connections = maybeConnections;
-            super(urlsOrConnections, FixedConnectionMap.fromArray(connections));
-            this.rawConnections = connections;
-        }
+    constructor({ urlReader, connections, }) {
+        super({
+            connections: FixedConnectionMap.fromArray(connections),
+            urlReader,
+        });
+        this.rawConnections = connections;
     }
 }
 exports.ConnectionRuntime = ConnectionRuntime;
 class SingleConnectionRuntime extends Runtime {
-    constructor(...params) {
-        let urlReader;
-        let connection;
-        let eventStream;
-        for (const param of params) {
-            if (isURLReader(param)) {
-                urlReader = param;
-            }
-            if (isConnection(param)) {
-                connection = param;
-            }
-            if (isEventStream(param)) {
-                eventStream = param;
-            }
-        }
-        if (connection === undefined) {
-            throw new Error('Expected connection to be passed into SingleConnectionRuntime');
-        }
-        super(urlReader, connection, eventStream);
+    constructor({ urlReader, connection, eventStream, cacheManager, }) {
+        super({
+            urlReader,
+            eventStream,
+            cacheManager,
+            connection,
+        });
         this.connection = connection;
     }
     get supportsNesting() {
@@ -1892,24 +1955,17 @@ class ModelMaterializer extends FluentState {
                     options = { ...options, testEnvironment: true };
                 }
             }
-            const parse = query instanceof URL
-                ? await Malloy.parse({
-                    url: query,
-                    urlReader,
-                    options,
-                })
-                : Malloy.parse({
-                    source: query,
-                    options,
-                });
+            const compilable = query instanceof URL ? { url: query } : { source: query };
             const model = await this.getModel();
             const queryModel = await Malloy.compile({
+                ...compilable,
                 urlReader,
                 connections,
-                parse,
                 model,
                 refreshSchemaCache,
                 noThrowOnError,
+                importBaseURL: options === null || options === void 0 ? void 0 : options.importBaseURL,
+                testEnvironment: options === null || options === void 0 ? void 0 : options.testEnvironment,
                 ...this.compileQueryOptions,
             });
             return queryModel.preparedQuery;
@@ -1934,24 +1990,17 @@ class ModelMaterializer extends FluentState {
         return new ModelMaterializer(this.runtime, async () => {
             const urlReader = this.runtime.urlReader;
             const connections = this.runtime.connections;
-            const parse = query instanceof URL
-                ? await Malloy.parse({
-                    url: query,
-                    urlReader,
-                    options,
-                })
-                : Malloy.parse({
-                    source: query,
-                    options,
-                });
+            const compilable = query instanceof URL ? { url: query } : { source: query };
             const model = await this.getModel();
             const queryModel = await Malloy.compile({
+                ...compilable,
                 urlReader,
                 connections,
-                parse,
                 model,
                 refreshSchemaCache: options === null || options === void 0 ? void 0 : options.refreshSchemaCache,
                 noThrowOnError: options === null || options === void 0 ? void 0 : options.noThrowOnError,
+                importBaseURL: options === null || options === void 0 ? void 0 : options.importBaseURL,
+                testEnvironment: options === null || options === void 0 ? void 0 : options.testEnvironment,
                 ...this.compileQueryOptions,
             });
             return queryModel;
@@ -2790,18 +2839,6 @@ class DataRecord extends Data {
     }
 }
 exports.DataRecord = DataRecord;
-function isURLReader(thing) {
-    return 'readURL' in thing;
-}
-function isLookupConnection(thing) {
-    return 'lookupConnection' in thing;
-}
-function isEventStream(thing) {
-    return 'emit' in thing;
-}
-function isConnection(thing) {
-    return 'runSQL' in thing;
-}
 class DataWriter {
     constructor(stream) {
         this.stream = stream;
@@ -3010,4 +3047,115 @@ class CSVWriter extends DataWriter {
     }
 }
 exports.CSVWriter = CSVWriter;
+class CacheManager {
+    constructor(modelCache) {
+        this.modelCache = modelCache;
+        this.modelDependencies = new Map();
+        this.modelInvalidationKeys = new Map();
+    }
+    async getCachedModelDef(urlReader, url) {
+        const _dependencies = this.modelDependencies.get(url);
+        if (_dependencies === undefined) {
+            return undefined;
+        }
+        const dependencies = [url, ...flatDeps(_dependencies)];
+        const invalidationKeys = {};
+        for (const dependency of dependencies) {
+            const invalidationKey = this.modelInvalidationKeys.get(dependency);
+            if (invalidationKey === undefined || invalidationKey === null) {
+                return undefined;
+            }
+            invalidationKeys[dependency] = invalidationKey;
+        }
+        for (const dependency of dependencies) {
+            const invalidationKey = await getInvalidationKey(urlReader, new URL(dependency));
+            if (invalidationKey !== invalidationKeys[dependency]) {
+                return undefined;
+            }
+        }
+        const cached = await this.modelCache.getModel(new URL(url));
+        if (cached === undefined) {
+            return undefined;
+        }
+        for (const dependency of dependencies) {
+            if (cached.invalidationKeys[dependency] !== invalidationKeys[dependency]) {
+                return undefined;
+            }
+        }
+        // Return the cached model def and the invalidation keys for this
+        // model def's dependencies
+        return { modelDef: cached.modelDef, invalidationKeys };
+    }
+    async setCachedModelDef(url, cachedModel) {
+        this.modelDependencies.set(url, cachedModel.modelDef.dependencies);
+        const invalidationKeys = {};
+        for (const dependency of [
+            url,
+            ...flatDeps(cachedModel.modelDef.dependencies),
+        ]) {
+            if (cachedModel.invalidationKeys[dependency] === null) {
+                return false;
+            }
+            if (cachedModel.invalidationKeys[dependency] === undefined) {
+                throw new Error(`Missing invalidation key for dependency ${dependency}`);
+            }
+            this.modelInvalidationKeys.set(dependency, cachedModel.invalidationKeys[dependency]);
+            invalidationKeys[dependency] = cachedModel.invalidationKeys[dependency];
+        }
+        const result = await this.modelCache.setModel(new URL(url), {
+            modelDef: cachedModel.modelDef,
+            invalidationKeys,
+        });
+        if (result) {
+            return true; // TODO just return `result` when it's a boolean
+        }
+        return false;
+    }
+}
+exports.CacheManager = CacheManager;
+function flatDeps(tree) {
+    return [...Object.keys(tree), ...Object.values(tree).map(flatDeps).flat()];
+}
+// TODO maybe make this memory bounded....
+class InMemoryModelCache {
+    constructor() {
+        this.models = new Map();
+    }
+    async getModel(url) {
+        return Promise.resolve(this.models.get(url.toString()));
+    }
+    async setModel(url, cachedModel) {
+        this.models.set(url.toString(), cachedModel);
+        return Promise.resolve(true);
+    }
+}
+exports.InMemoryModelCache = InMemoryModelCache;
+function hashForInvalidationKey(input) {
+    const MALLOY_UUID = '76c17e9d-f3ce-5f2d-bfde-98ad3d2a37f6';
+    return (0, uuid_1.v5)(input, MALLOY_UUID);
+}
+async function readURL(urlReader, url) {
+    const result = await urlReader.readURL(url);
+    const { contents, invalidationKey } = typeof result === 'string'
+        ? { contents: result, invalidationKey: undefined }
+        : result;
+    return {
+        contents,
+        invalidationKey: isInternalURL(url.toString())
+            ? null
+            : invalidationKey !== null && invalidationKey !== void 0 ? invalidationKey : hashForInvalidationKey(contents),
+    };
+}
+async function getInvalidationKey(urlReader, url) {
+    if (isInternalURL(url.toString())) {
+        return null;
+    }
+    if (urlReader.getInvalidationKey !== undefined) {
+        return await urlReader.getInvalidationKey(url);
+    }
+    return (await readURL(urlReader, url)).invalidationKey;
+}
+function isInternalURL(url) {
+    return url.startsWith('internal://');
+}
 //# sourceMappingURL=malloy.js.map

package/dist/runtime_types.d.ts

@@ -14,6 +14,7 @@ export type ModelURL = URL;
  * A URL whose contents is a Malloy query.
  */
 export type QueryURL = URL;
+export type InvalidationKey = string | number | Date | null;
 /**
  * An object capable of reading the contents of a URL in some context.
  */
@@ -24,7 +25,11 @@ export interface URLReader {
      * @param url The URL to read.
      * @return A promise to the contents of the URL.
      */
-    readURL: (url: URL) => Promise<string>;
+    readURL: (url: URL) => Promise<string | {
+        contents: string;
+        invalidationKey?: InvalidationKey;
+    }>;
+    getInvalidationKey?: (url: URL) => Promise<InvalidationKey>;
 }
 export interface EventStream {
     emit(id: string, data: any): void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.232-dev250118020814",
+  "version": "0.0.232-dev250127221938",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",