@cubejs-backend/schema-compiler 1.7.47 → 1.7.49

package/dist/src/compiler/DataSchemaCompiler.js

@@ -18,7 +18,190 @@ const native_1 = require("@cubejs-backend/native");
 const UserError_1 = require("./UserError");
 const ErrorReporter_1 = require("./ErrorReporter");
 const CubeSymbols_1 = require("./CubeSymbols");
+const transpilers_1 = require("./transpilers");
 const ctxFileStorage = new async_hooks_1.AsyncLocalStorage();
+// Set while the function passed to `memo` runs, including its async continuations
+const memoFnStorage = new async_hooks_1.AsyncLocalStorage();
+// For memo() keys: objects compare by their keys whatever the property order, and NaN and
+// Infinity don't encode as null
+const memoKeyReplacer = (_key, value) => {
+    if (typeof value === 'number' && !Number.isFinite(value)) {
+        return { $memoNumber: String(value) };
+    }
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+        return Object.fromEntries(Object.keys(value).sort().map((k) => [k, value[k]]));
+    }
+    return value;
+};
+// Shared realm: closures bind the running compile's scope lexically through `with`, so lazy reads
+// (sql reading COMPILE_CONTEXT) keep resolving it after the compile; the inner function keeps
+// top-level declarations off the shared global and gets the compile's own global as `this`.
+const SHARED_SCOPE_GETTER = '__cubejsCompileScope';
+const sharedScriptPrefix = (strict) => `with (${SHARED_SCOPE_GETTER}(${strict})) { (function () {`;
+const SHARED_SCRIPT_SUFFIX = '\n}).call(globalThis); }';
+// Whether the source opens with a 'use strict' directive, after whitespace and comments.
+// A linear scan: a regex over repeated comments can backtrack exponentially.
+function hasUseStrictDirective(source) {
+    let i = 0;
+    while (i < source.length) {
+        if (/\s/.test(source[i])) {
+            i++;
+        }
+        else if (source.startsWith('//', i)) {
+            const end = source.indexOf('\n', i);
+            if (end === -1)
+                return false;
+            i = end + 1;
+        }
+        else if (source.startsWith('/*', i)) {
+            const end = source.indexOf('*/', i + 2);
+            if (end === -1)
+                return false;
+            i = end + 2;
+        }
+        else {
+            break;
+        }
+    }
+    return source.startsWith("'use strict'", i) || source.startsWith('"use strict"', i);
+}
+// UMD wrappers probe these with `typeof`: read as undefined, not as a ReferenceError
+const PROBED_GLOBALS = new Set(['module', 'exports', 'define']);
+const sharedScopeStorage = new async_hooks_1.AsyncLocalStorage();
+let sharedVmContext = null;
+let sharedRealmGlobal = null;
+const getSharedVmContext = () => {
+    if (!sharedVmContext) {
+        const sandbox = {};
+        Object.defineProperty(sandbox, SHARED_SCOPE_GETTER, {
+            value: (strict) => {
+                const compile = sharedScopeStorage.getStore();
+                if (!compile) {
+                    throw new Error('No compile scope stored in context');
+                }
+                return strict ? compile.strictScope : compile.scope;
+            },
+            writable: false,
+            enumerable: false,
+            configurable: false,
+        });
+        const refuse = (key) => {
+            // The realm's own TypeError, as model code would catch it
+            throw new sharedRealmGlobal.TypeError(`Cannot set global '${String(key)}': data models share one realm with CUBEJS_COMPILER_MULTI_TENANT_SHARING`);
+        };
+        // What still reaches the real global (`this` of a nested sloppy function) fails loudly: a
+        // value stored there would be every tenant's.
+        sharedVmContext = vm_1.default.createContext(new Proxy(sandbox, {
+            set: (_t, key) => refuse(key),
+            defineProperty: (_t, key) => refuse(key),
+            deleteProperty: (_t, key) => refuse(key),
+        }));
+        sharedRealmGlobal = vm_1.default.runInContext('globalThis', sharedVmContext);
+    }
+    return sharedVmContext;
+};
+// Names the compile assigns (implicit globals, `globalThis.x`, `this.x`, even `JSON = ...`) land
+// in `vars`; reads fall back to the realm's built-ins. Unknown names throw a ReferenceError, also
+// under `typeof` (except PROBED_GLOBALS).
+const createSharedCompileScope = (globals) => {
+    getSharedVmContext();
+    const realm = sharedRealmGlobal;
+    const vars = Object.assign(Object.create(null), globals);
+    // Built-ins this compile deleted from its global
+    const deleted = new Set();
+    const builtIn = (key) => key !== SHARED_SCOPE_GETTER && !deleted.has(key) && key in realm;
+    const read = (key) => {
+        if (key in vars) {
+            return vars[key];
+        }
+        if (builtIn(key)) {
+            return realm[key];
+        }
+        if (typeof key !== 'string' || PROBED_GLOBALS.has(key)) {
+            return undefined;
+        }
+        throw new realm.ReferenceError(`${key} is not defined`);
+    };
+    const remove = (key) => {
+        const removed = Reflect.deleteProperty(vars, key);
+        if (removed && builtIn(key)) {
+            deleted.add(key);
+        }
+        return removed;
+    };
+    // The compile's `globalThis` and top-level `this`: its own properties are `vars` (the target, so
+    // defining, deleting and reflecting on them behave as on an ordinary object), then the built-ins
+    const realmOwn = (key) => builtIn(key) && Object.prototype.hasOwnProperty.call(realm, key);
+    const compileGlobal = new Proxy(vars, {
+        get: (t, key) => (key in t || builtIn(key) ? read(key) : undefined),
+        has: (t, key) => key in t || builtIn(key),
+        // Without it, OrdinarySet would copy the built-in's attributes through getOwnPropertyDescriptor
+        set: (t, key, value) => {
+            t[key] = value;
+            deleted.delete(key);
+            return true;
+        },
+        defineProperty: (t, key, descriptor) => {
+            // Redefining a built-in keeps the attributes the descriptor leaves out, as on a global
+            const base = !Object.prototype.hasOwnProperty.call(t, key) && realmOwn(key)
+                ? { ...Reflect.getOwnPropertyDescriptor(realm, key), configurable: true }
+                : {};
+            const accessor = 'get' in descriptor || 'set' in descriptor;
+            const merged = accessor || 'get' in base
+                ? { enumerable: base.enumerable, configurable: base.configurable, ...descriptor }
+                : { ...base, ...descriptor };
+            deleted.delete(key);
+            return Reflect.defineProperty(t, key, merged);
+        },
+        deleteProperty: (_t, key) => remove(key),
+        ownKeys: (t) => [...new Set([
+                ...Reflect.ownKeys(t),
+                ...Reflect.ownKeys(realm).filter((key) => realmOwn(key)),
+            ])],
+        getOwnPropertyDescriptor: (t, key) => {
+            if (Object.prototype.hasOwnProperty.call(t, key) || !realmOwn(key)) {
+                return Reflect.getOwnPropertyDescriptor(t, key);
+            }
+            // A built-in the compile hasn't shadowed: reported configurable, as the target lacks it
+            return { ...Reflect.getOwnPropertyDescriptor(realm, key), configurable: true };
+        },
+    });
+    Object.defineProperty(vars, 'globalThis', { value: compileGlobal, writable: true, configurable: true, enumerable: false });
+    const scopeTraps = {
+        get: (_t, key) => read(key),
+        set: (t, key, value) => {
+            t[key] = value;
+            deleted.delete(key);
+            return true;
+        },
+        deleteProperty: (_t, key) => remove(key),
+    };
+    const scope = new Proxy(vars, {
+        ...scopeTraps,
+        has: (_t, key) => typeof key === 'string' && key !== SHARED_SCOPE_GETTER,
+    });
+    const strictScope = new Proxy(vars, {
+        ...scopeTraps,
+        has: (t, key) => typeof key === 'string' && (key in t || builtIn(key)),
+    });
+    // `new Function(...)` / `Function(...)` compile their body in this compile's scope, as they would
+    // at the global scope of a realm of its own. The result is an ordinary function of the realm.
+    const RealmFunction = realm.Function;
+    const compileFunction = function Function(...args) {
+        const body = args.length > 0 ? String(args[args.length - 1]) : '';
+        const params = args.slice(0, -1).map(String).join(',');
+        // Parse params and body on their own first, so malformed input throws the realm's SyntaxError
+        // instead of closing the wrapper below early.
+        RealmFunction(...args);
+        const factory = RealmFunction('scope', `with (scope) { return function anonymous(${params}\n) {\n${body}\n}; }`);
+        return factory(hasUseStrictDirective(body) ? strictScope : scope);
+    };
+    compileFunction.prototype = RealmFunction.prototype;
+    Object.setPrototypeOf(compileFunction, RealmFunction.prototype);
+    Object.defineProperty(compileFunction, 'length', { value: 1 });
+    Object.defineProperty(vars, 'Function', { value: compileFunction, writable: true, configurable: true, enumerable: false });
+    return { scope, strictScope, vars };
+};
 const NATIVE_IS_SUPPORTED = (0, shared_1.isNativeSupported)();
 const moduleFileCache = {};
 const JINJA_SYNTAX = /{%|%}|{{|}}/ig;
@@ -87,6 +270,9 @@ class DataSchemaCompiler {
     compiledYamlCache;
     compiledJinjaCache;
     compileV8ContextCache = null;
+    sharedVmContext;
+    // Flag on: per-compile `with` scope evaluated inside the shared realm.
+    compileScope = null;
     // FIXME: Is public only because of tests, should be private
     compilePromise;
     currentQuery;
@@ -121,6 +307,7 @@ class DataSchemaCompiler {
         this.compiledScriptCache = options.compiledScriptCache;
         this.compiledYamlCache = options.compiledYamlCache;
         this.compiledJinjaCache = options.compiledJinjaCache;
+        this.sharedVmContext = !!options.sharedVmContext;
     }
     compileObjects(compileServices, objects, errorsReport) {
         try {
@@ -251,6 +438,22 @@ class DataSchemaCompiler {
         let compiledFiles = {};
         let asyncModules = [];
         let transpiledFiles = [];
+        // Results of `memo` calls. Unlike the objects above it survives between the compile stages:
+        // the model is evaluated once per stage, and `memo` makes it do a costly call (an API request
+        // from an `asyncModule`, say) only once per compile.
+        const memoResults = new Map();
+        // Stage that last looked up each key MemoKeyTranspiler generated
+        const memoCallSiteStages = new Map();
+        let memoStage = -1;
+        // Model objects go to the current stage only, while a memoized function runs in the first stage
+        // that asks for its key: objects defined from there would be missing from all later stages.
+        const assertOutsideMemo = (globalName) => {
+            const memoCall = memoFnStorage.getStore();
+            if (memoCall !== undefined) {
+                throw new UserError_1.UserError(`${globalName}() can't be called from the function passed to ${memoCall}: ` +
+                    'return the data from memo() and define model objects outside of it');
+            }
+        };
         const cleanup = () => {
             cubes = [];
             exports = {};
@@ -259,12 +462,15 @@ class DataSchemaCompiler {
             compiledFiles = {};
             asyncModules = [];
         };
-        this.compileV8ContextCache = vm_1.default.createContext({
+        const compileGlobals = {
             view: (name, cube) => {
                 const file = ctxFileStorage.getStore();
                 if (!file) {
                     throw new Error('No file stored in context');
                 }
+                if (cube) {
+                    assertOutsideMemo('view');
+                }
                 return !cube ?
                     this.cubeFactory({ ...name, fileName: file.fileName, isView: true }) :
                     cubes.push({ ...cube, name, fileName: file.fileName, isView: true });
@@ -274,6 +480,9 @@ class DataSchemaCompiler {
                 if (!file) {
                     throw new Error('No file stored in context');
                 }
+                if (cube) {
+                    assertOutsideMemo('cube');
+                }
                 return !cube ?
                     this.cubeFactory({ ...name, fileName: file.fileName }) :
                     cubes.push({ ...cube, name, fileName: file.fileName });
@@ -283,6 +492,7 @@ class DataSchemaCompiler {
                 if (!file) {
                     throw new Error('No file stored in context');
                 }
+                assertOutsideMemo('context');
                 return contexts.push({ ...context, name, fileName: file.fileName });
             },
             view_group: (name, viewGroup) => {
@@ -290,8 +500,14 @@ class DataSchemaCompiler {
                 if (!file) {
                     throw new Error('No file stored in context');
                 }
+                assertOutsideMemo('view_group');
                 viewGroups.push({ ...viewGroup, name, fileName: file.fileName });
-                this.compileV8ContextCache[name] = name;
+                if (this.sharedVmContext) {
+                    this.compileScope.vars[name] = name;
+                }
+                else {
+                    this.compileV8ContextCache[name] = name;
+                }
                 return name;
             },
             addExport: (obj) => {
@@ -314,11 +530,65 @@ class DataSchemaCompiler {
                 if (!file) {
                     throw new Error('No file stored in context');
                 }
+                assertOutsideMemo('asyncModule');
                 // We need to run async module code in the context of the original data model file
                 // where it was defined. So we pass the same file to the async context.
                 // @see https://nodejs.org/api/async_context.html#class-asynclocalstorage
                 asyncModules.push(async () => ctxFileStorage.run(file, () => fn()));
             },
+            memo: (key, fn) => {
+                if (typeof key === 'function' && fn === undefined) {
+                    // MemoKeyTranspiler gives `memo(fn)` its key: this call didn't go through it
+                    throw new Error('memo() expects a key as its first argument: memo(key, fn)');
+                }
+                if (typeof fn !== 'function') {
+                    throw new Error('memo() expects a function as its second argument');
+                }
+                // Strings are encoded too, or `memo('1', ...)` would share the result of `memo(1, ...)`
+                let cacheKey;
+                try {
+                    cacheKey = JSON.stringify(key, memoKeyReplacer);
+                }
+                catch {
+                    // BigInt or a cyclic structure
+                }
+                if (typeof cacheKey !== 'string') {
+                    throw new Error('memo() expects a string or a JSON-serializable key as its first argument');
+                }
+                const callSite = transpilers_1.MemoKeyTranspiler.callSiteOf(key);
+                // For errors: the key as the code would write it
+                const label = callSite !== undefined ? `memo() at ${callSite}` : `memo(${cacheKey})`;
+                if (memoStage < 0 || !ctxFileStorage.getStore()) {
+                    // Called from a member (`sql: () => memo(...)`) evaluated by a compiler or a query, not
+                    // from a model file: it may run any number of times, so there's no stage to cache for
+                    return memoFnStorage.run(label, fn);
+                }
+                if (callSite !== undefined) {
+                    // A top-level call site runs once per stage: twice means it's in a function or a loop,
+                    // where the one key would give every call the result of the first one
+                    if (memoCallSiteStages.get(cacheKey) === memoStage) {
+                        throw new UserError_1.UserError(`memo() at ${callSite} is called more than once per compile stage: as it's in a function or a loop, ` +
+                            'its result may depend on the arguments: pass a key that does, e.g. memo([\'table\', name], fn)');
+                    }
+                    memoCallSiteStages.set(cacheKey, memoStage);
+                }
+                let result = memoResults.get(cacheKey);
+                if (!result) {
+                    // A promise is stored as is, so concurrent and later calls share one invocation.
+                    // A throw is stored too: every stage has to see the same outcome.
+                    try {
+                        result = { value: memoFnStorage.run(label, fn) };
+                    }
+                    catch (error) {
+                        result = { error };
+                    }
+                    memoResults.set(cacheKey, result);
+                }
+                if ('error' in result) {
+                    throw result.error;
+                }
+                return result.value;
+            },
             require: (extensionName) => {
                 const file = ctxFileStorage.getStore();
                 if (!file) {
@@ -346,10 +616,17 @@ class DataSchemaCompiler {
                 }
             },
             COMPILE_CONTEXT: this.standalone ? this.standaloneCompileContextProxy() : this.cloneCompileContextWithGetterAlias(this.compileContext || {}),
-        });
+        };
+        if (this.sharedVmContext) {
+            this.compileScope = createSharedCompileScope(compileGlobals);
+        }
+        else {
+            this.compileV8ContextCache = vm_1.default.createContext(compileGlobals);
+        }
         const compilePhaseFirst = async (compilers, stage) => {
             // clear the objects for the next phase
             cleanup();
+            memoStage = stage;
             transpiledFiles = await transpilePhaseFirst(stage);
             // We render jinja and transpile yaml only once on first phase and then use resulting JS for these files
             // afterward avoiding costly YAML/Python parsing again. Original JS files are preserved as is for cache hits.
@@ -360,10 +637,11 @@ class DataSchemaCompiler {
         const compilePhase = async (compilers, stage) => {
             // clear the objects for the next phase
             cleanup();
+            memoStage = stage;
             transpiledFiles = await transpilePhase(stage);
             return this.compileCubeFiles(cubes, contexts, viewGroups, compiledFiles, asyncModules, compilers, transpiledFiles, errorsReport);
         };
-        return compilePhaseFirst({ cubeCompilers: this.cubeNameCompilers }, 0)
+        const compileAll = () => compilePhaseFirst({ cubeCompilers: this.cubeNameCompilers }, 0)
             .then(() => compilePhase({ cubeCompilers: this.preTranspileCubeCompilers.concat([this.viewCompilationGate]) }, 1))
             .then(() => (this.viewCompilationGate.shouldCompileViews() ?
             compilePhase({ cubeCompilers: this.viewCompilers }, 2)
@@ -379,6 +657,12 @@ class DataSchemaCompiler {
             cleanup();
             transpiledFiles = [];
             toCompile = [];
+            // The functions put into the compile context (cube, view, require, ...) close over
+            // this scope, and member functions keep that context alive as long as the compiled
+            // model lives: drop the model files' contents here, or they stay retained with it.
+            originalJsFiles.length = 0;
+            jinjaTemplatedFiles.length = 0;
+            yamlFiles.length = 0;
             if (transpilationNative) {
                 // Clean up cache
                 const dummyFile = {
@@ -390,7 +674,18 @@ class DataSchemaCompiler {
             else if (this.workerPool) {
                 this.workerPool.terminate();
             }
+        })
+            // A failed compile mustn't keep the fetched data either
+            .finally(() => {
+            memoStage = -1;
+            memoResults.clear();
+            memoCallSiteStages.clear();
+            // These hold the globals.py TemplateContext, where the Python @memo caches live
+            this.pythonContext = null;
+            this.yamlCompiler.free();
         });
+        // Every continuation of the compile (asyncModule callbacks included) sees its scope
+        return this.compileScope ? sharedScopeStorage.run(this.compileScope, compileAll) : compileAll();
     }
     compile() {
         if (!this.compilePromise) {
@@ -400,6 +695,7 @@ class DataSchemaCompiler {
                 }
                 // Free unneeded resources
                 this.compileV8ContextCache = null;
+                this.compileScope = null;
                 this.cubeDictionary.free();
                 this.cubeOnlySymbols.free();
                 this.cubeAndViewSymbols.free();
@@ -625,6 +921,8 @@ class DataSchemaCompiler {
         return undefined;
     }
     async transpileYamlFile(file, errorsReport, { cubeNames, cubeSymbols, compilerId, jinjaUsed }) {
+        // The YAML to JS output only depends on the content: YAML is transpiled once, in the
+        // first phase, before any cube symbols are known. So the key is safe to share between apps.
         const cacheKey = crypto_1.default.createHash('md5').update(file.content).digest('hex');
         if (this.compiledYamlCache.has(cacheKey)) {
             const content = this.compiledYamlCache.get(cacheKey);
@@ -643,7 +941,10 @@ class DataSchemaCompiler {
             errorsReport.addErrors(res[0].errors, file.fileName);
             errorsReport.addWarnings(res[0].warnings);
             errorsReport.exitFile();
-            this.compiledYamlCache.set(cacheKey, res[0].code);
+            // A hit doesn't replay errors, so a failed transpilation must not be cached
+            if (!res[0].errors?.length) {
+                this.compiledYamlCache.set(cacheKey, res[0].code);
+            }
             return { ...file, content: res[0].code };
         }
         else {
@@ -658,7 +959,9 @@ class DataSchemaCompiler {
             const res = await this.workerPool.exec('transpileYaml', [data]);
             errorsReport.addErrors(res.errors, file.fileName);
             errorsReport.addWarnings(res.warnings);
-            this.compiledYamlCache.set(cacheKey, res.content);
+            if (!res.errors?.length) {
+                this.compiledYamlCache.set(cacheKey, res.content);
+            }
             return { ...file, content: res.content };
         }
     }
@@ -719,7 +1022,12 @@ class DataSchemaCompiler {
         this.compileJsFile(file, errorsReport, { doSyntaxCheck });
     }
     getJsScript(file) {
-        const cacheKey = crypto_1.default.createHash('md5').update(file.content).digest('hex');
+        // The script carries its file name (stack traces), and the cache may be shared between apps
+        const cacheKey = crypto_1.default.createHash('md5')
+            .update(file.fileName)
+            .update('\0')
+            .update(file.content)
+            .digest('hex');
         if (this.compiledScriptCache.has(cacheKey)) {
             return this.compiledScriptCache.get(cacheKey);
         }
@@ -727,6 +1035,25 @@ class DataSchemaCompiler {
         this.compiledScriptCache.set(cacheKey, script);
         return script;
     }
+    getSharedJsScript(file) {
+        // Not the source getJsScript compiles (the wrapper), hence the prefix
+        const cacheKey = `shared:${crypto_1.default.createHash('md5')
+            .update(file.fileName)
+            .update('\0')
+            .update(file.content)
+            .digest('hex')}`;
+        if (this.compiledScriptCache.has(cacheKey)) {
+            return this.compiledScriptCache.get(cacheKey);
+        }
+        // The prefix is on line 1, so a negative column offset keeps positions in stack traces intact.
+        const prefix = sharedScriptPrefix(hasUseStrictDirective(file.content));
+        const script = new vm_1.default.Script(`${prefix}${file.content}${SHARED_SCRIPT_SUFFIX}`, {
+            filename: file.fileName,
+            columnOffset: -prefix.length,
+        });
+        this.compiledScriptCache.set(cacheKey, script);
+        return script;
+    }
     compileJsFile(file, errorsReport, { doSyntaxCheck } = { doSyntaxCheck: false }) {
         if (doSyntaxCheck) {
             // There is no need to run syntax check for data model files
@@ -738,6 +1065,16 @@ class DataSchemaCompiler {
             }
         }
         try {
+            if (this.sharedVmContext) {
+                const script = this.getSharedJsScript(file);
+                const scope = this.compileScope;
+                ctxFileStorage.run(file, () => {
+                    sharedScopeStorage.run(scope, () => {
+                        script.runInContext(getSharedVmContext(), { timeout: 15000 });
+                    });
+                });
+                return;
+            }
             const script = this.getJsScript(file);
             // We use AsyncLocalStorage to store the current file context
             // so that it can be accessed in the script execution context even within async functions.