npm - @directus/api - Versions diffs - 13.2.0 → 14.0.0 - Mend

@directus/api 13.2.0 → 14.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (101) hide show

package/dist/extensions/manager.js CHANGED Viewed

@@ -8,9 +8,9 @@ import nodeResolveDefault from '@rollup/plugin-node-resolve';
 import virtualDefault from '@rollup/plugin-virtual';
 import chokidar, { FSWatcher } from 'chokidar';
 import express, { Router } from 'express';
+import ivm from 'isolated-vm';
 import { clone } from 'lodash-es';
-import { readdir } from 'node:fs/promises';
-import { createRequire } from 'node:module';
+import { readFile, readdir } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import path from 'path';
@@ -21,50 +21,96 @@ import env from '../env.js';
 import { getFlowManager } from '../flows.js';
 import logger from '../logger.js';
 import * as services from '../services/index.js';
+import { deleteFromRequireCache } from '../utils/delete-from-require-cache.js';
 import getModuleDefault from '../utils/get-module-default.js';
 import { getSchema } from '../utils/get-schema.js';
+import { importFileUrl } from '../utils/import-file-url.js';
 import { JobQueue } from '../utils/job-queue.js';
 import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js';
-import { getExtensions } from './get-extensions.js';
-import { getSharedDepsMapping } from './get-shared-deps-mapping.js';
-import { normalizeExtensionInfo } from './normalize-extension-info.js';
-import { wrapEmbeds } from './wrap-embeds.js';
+import { getExtensionsSettings } from './lib/get-extensions-settings.js';
+import { getExtensions } from './lib/get-extensions.js';
+import { getSharedDepsMapping } from './lib/get-shared-deps-mapping.js';
+import { generateApiExtensionsSandboxEntrypoint } from './lib/sandbox/generate-api-extensions-sandbox-entrypoint.js';
+import { instantiateSandboxSdk } from './lib/sandbox/sdk/instantiate.js';
+import { wrapEmbeds } from './lib/wrap-embeds.js';
 // Workaround for https://github.com/rollup/plugins/issues/1329
 const virtual = virtualDefault;
 const alias = aliasDefault;
 const nodeResolve = nodeResolveDefault;
-const require = createRequire(import.meta.url);
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const defaultOptions = {
-    schedule: true,
-    watch: env['EXTENSIONS_AUTO_RELOAD'] && env['NODE_ENV'] !== 'development',
-};
 export class ExtensionManager {
+    options = {
+        schedule: true,
+        watch: env['EXTENSIONS_AUTO_RELOAD'] && env['NODE_ENV'] !== 'development',
+    };
+    /**
+     * Whether or not the extensions have been read from disk and registered into the system
+     */
     isLoaded = false;
-    options;
+    /**
+     * All extensions that are loaded within the current process
+     */
     extensions = [];
-    appExtensions = null;
-    appExtensionChunks;
-    apiExtensions = [];
-    apiEmitter;
-    hookEvents = [];
-    endpointRouter;
+    /**
+     * Settings for the extensions that are loaded within the current process
+     */
+    extensionsSettings = [];
+    /**
+     * App extensions rolled up into a single bundle. Any chunks from the bundle will be available
+     * under appExtensionChunks
+     */
+    appExtensionsBundle = null;
+    /**
+     * Individual filename chunks from the rollup bundle. Used to improve the performance by allowing
+     * extensions to split up their bundle into multiple smaller chunks
+     */
+    appExtensionChunks = new Map();
+    /**
+     * Callbacks to be able to unregister extensions
+     */
+    unregisterFunctionMap = new Map();
+    /**
+     * A local-to-extensions scoped emitter that can be used to fire and listen to custom events
+     * between extensions. These events are completely isolated from the core events that trigger
+     * hooks etc
+     */
+    localEmitter = new Emitter();
+    /**
+     * Locally scoped express router used for custom endpoints. Allows extensions to dynamically
+     * register and de-register endpoints without affecting the regular global router
+     */
+    endpointRouter = Router();
+    /**
+     * Custom HTML to be injected at the end of the `<head>` tag of the app's index.html
+     */
     hookEmbedsHead = [];
+    /**
+     * Custom HTML to be injected at the end of the `<body>` tag of the app's index.html
+     */
     hookEmbedsBody = [];
-    reloadQueue;
+    /**
+     * Used to prevent race conditions when reloading extensions. Forces each reload to happen in
+     * sequence.
+     */
+    reloadQueue = new JobQueue();
+    /**
+     * Optional file system watcher to auto-reload extensions when the local file system changes
+     */
     watcher = null;
-    constructor() {
-        this.options = defaultOptions;
-        this.apiEmitter = new Emitter();
-        this.endpointRouter = Router();
-        this.reloadQueue = new JobQueue();
-        this.appExtensionChunks = new Map();
-    }
+    /**
+     * Load and register all extensions
+     *
+     * @param {ExtensionManagerOptions} options - Extension manager configuration options
+     * @param {boolean} options.schedule - Whether or not to allow for scheduled (CRON) hook extensions
+     * @param {boolean} options.watch - Whether or not to watch the local extensions folder for changes
+     */
     async initialize(options = {}) {
-        this.options = {
-            ...defaultOptions,
-            ...options,
-        };
+        if (options.schedule !== undefined) {
+            this.options.schedule = options.schedule;
+        }
+        if (options.watch !== undefined) {
+            this.options.watch = options.watch;
+        }
         const wasWatcherInitialized = this.watcher !== null;
         if (this.options.watch && !wasWatcherInitialized) {
             this.initializeWatcher();
@@ -74,15 +120,48 @@ export class ExtensionManager {
         }
         if (!this.isLoaded) {
             await this.load();
-            const loadedExtensions = this.getExtensionsList();
-            if (loadedExtensions.length > 0) {
-                logger.info(`Loaded extensions: ${loadedExtensions.map((ext) => ext.name).join(', ')}`);
+            if (this.extensions.length > 0) {
+                logger.info(`Loaded extensions: ${this.extensions.map((ext) => ext.name).join(', ')}`);
             }
         }
         if (this.options.watch && !wasWatcherInitialized) {
             this.updateWatchedExtensions(this.extensions);
         }
     }
+    /**
+     * Load all extensions from disk and register them in their respective places
+     */
+    async load() {
+        try {
+            await ensureExtensionDirs(env['EXTENSIONS_PATH'], NESTED_EXTENSION_TYPES);
+            this.extensions = await getExtensions();
+            this.extensionsSettings = await getExtensionsSettings(this.extensions);
+        }
+        catch (err) {
+            logger.warn(`Couldn't load extensions`);
+            logger.warn(err);
+        }
+        await this.registerHooks();
+        await this.registerEndpoints();
+        await this.registerOperations();
+        await this.registerBundles();
+        if (env['SERVE_APP']) {
+            this.appExtensionsBundle = await this.generateExtensionBundle();
+        }
+        this.isLoaded = true;
+    }
+    /**
+     * Unregister all extensions from the current process
+     */
+    async unload() {
+        await this.unregisterApiExtensions();
+        this.localEmitter.offAll();
+        this.appExtensionsBundle = null;
+        this.isLoaded = false;
+    }
+    /**
+     * Reload all the extensions. Will unload if extensions have already been loaded
+     */
     reload() {
         this.reloadQueue.enqueue(async () => {
             if (this.isLoaded) {
@@ -107,57 +186,42 @@ export class ExtensionManager {
             }
         });
     }
-    getExtensionsList(type) {
-        const extensionInfo = this.extensions.map(normalizeExtensionInfo);
-        if (type) {
-            return extensionInfo.filter((extension) => extension.type === type);
-        }
-        return extensionInfo;
-    }
-    getExtension(name) {
-        return this.extensions.find((extension) => extension.name === name);
-    }
-    getAppExtensions() {
-        return this.appExtensions;
+    /**
+     * Return the previously generated app extensions bundle
+     */
+    getAppExtensionsBundle() {
+        return this.appExtensionsBundle;
     }
+    /**
+     * Return the previously generated app extension bundle chunk by name
+     */
     getAppExtensionChunk(name) {
         return this.appExtensionChunks.get(name) ?? null;
     }
+    /**
+     * Return the scoped router for custom endpoints
+     */
     getEndpointRouter() {
         return this.endpointRouter;
     }
+    /**
+     * Return the custom HTML head and body embeds wrapped in a marker comment
+     */
     getEmbeds() {
         return {
             head: wrapEmbeds('Custom Embed Head', this.hookEmbedsHead),
             body: wrapEmbeds('Custom Embed Body', this.hookEmbedsBody),
         };
     }
-    async load() {
-        try {
-            await ensureExtensionDirs(env['EXTENSIONS_PATH'], NESTED_EXTENSION_TYPES);
-            this.extensions = await getExtensions();
-        }
-        catch (err) {
-            logger.warn(`Couldn't load extensions`);
-            logger.warn(err);
-        }
-        await this.registerHooks();
-        await this.registerEndpoints();
-        await this.registerOperations();
-        await this.registerBundles();
-        if (env['SERVE_APP']) {
-            this.appExtensions = await this.generateExtensionBundle();
-        }
-        this.isLoaded = true;
-    }
-    async unload() {
-        await this.unregisterApiExtensions();
-        this.apiEmitter.offAll();
-        if (env['SERVE_APP']) {
-            this.appExtensions = null;
-        }
-        this.isLoaded = false;
+    /**
+     * Allow reading the installed extensions
+     */
+    getExtensions() {
+        return this.extensions;
     }
+    /**
+     * Start the chokidar watcher for extensions on the local filesystem
+     */
     initializeWatcher() {
         logger.info('Watching extensions for changes...');
         const extensionDirUrl = pathToRelativeUrl(env['EXTENSIONS_PATH']);
@@ -181,12 +245,19 @@ export class ExtensionManager {
             .on('change', () => this.reload())
             .on('unlink', () => this.reload());
     }
+    /**
+     * Close and destroy the local filesystem watcher if enabled
+     */
     async closeWatcher() {
         if (this.watcher) {
             await this.watcher.close();
             this.watcher = null;
         }
     }
+    /**
+     * Update the chokidar watcher configuration when new extensions are added or existing ones
+     * removed
+     */
     updateWatchedExtensions(added, removed = []) {
         if (this.watcher) {
             const toPackageExtensionPaths = (extensions) => extensions
@@ -203,6 +274,10 @@ export class ExtensionManager {
             this.watcher.unwatch(removedPackageExtensionPaths);
         }
     }
+    /**
+     * Uses rollup to bundle the app extensions together into a single file the app can download and
+     * run.
+     */
     async generateExtensionBundle() {
         const sharedDepsMapping = await getSharedDepsMapping(APP_SHARED_DEPS);
         const internalImports = Object.entries(sharedDepsMapping).map(([name, path]) => ({
@@ -232,15 +307,66 @@ export class ExtensionManager {
         }
         return null;
     }
+    async registerSandboxedApiExtension(extension) {
+        const sandboxMemory = Number(env['EXTENSIONS_SANDBOX_MEMORY']);
+        const sandboxTimeout = Number(env['EXTENSIONS_SANDBOX_TIMEOUT']);
+        const entrypointPath = path.resolve(extension.path, isTypeIn(extension, HYBRID_EXTENSION_TYPES) ? extension.entrypoint.api : extension.entrypoint);
+        const extensionCode = await readFile(entrypointPath, 'utf-8');
+        const isolate = new ivm.Isolate({
+            memoryLimit: sandboxMemory,
+            onCatastrophicError: (e) => {
+                logger.error(`Error in API extension sandbox of ${extension.type} "${extension.name}"`);
+                logger.error(e);
+                process.abort();
+            },
+        });
+        const context = await isolate.createContext();
+        const module = await isolate.compileModule(extensionCode, { filename: `file://${entrypointPath}` });
+        const sdkModule = await instantiateSandboxSdk(isolate, extension.sandbox?.requestedScopes ?? {});
+        await module.instantiate(context, (specifier) => {
+            if (specifier !== 'directus:api') {
+                throw new Error('Imports other than "directus:api" are prohibited in API extension sandboxes');
+            }
+            return sdkModule;
+        });
+        await module.evaluate({ timeout: sandboxTimeout });
+        const cb = await module.namespace.get('default', { reference: true });
+        const { code, hostFunctions, unregisterFunction } = generateApiExtensionsSandboxEntrypoint(extension.type, extension.name, this.endpointRouter);
+        await context.evalClosure(code, [cb, ...hostFunctions.map((fn) => new ivm.Reference(fn))], {
+            timeout: sandboxTimeout,
+            filename: '<extensions-sandbox>',
+        });
+        this.unregisterFunctionMap.set(extension.name, async () => {
+            await unregisterFunction();
+            isolate.dispose();
+        });
+    }
+    /**
+     * Import the hook module code for all hook extensions, and register them individually through
+     * registerHook
+     */
     async registerHooks() {
         const hooks = this.extensions.filter((extension) => extension.type === 'hook');
         for (const hook of hooks) {
+            const { enabled } = this.extensionsSettings.find(({ name }) => name === hook.name) ?? { enabled: false };
+            if (!enabled)
+                continue;
             try {
-                const hookPath = path.resolve(hook.path, hook.entrypoint);
-                const hookInstance = await import(`./${pathToRelativeUrl(hookPath, __dirname)}?t=${Date.now()}`);
-                const config = getModuleDefault(hookInstance);
-                this.registerHook(config, hook.name);
-                this.apiExtensions.push({ path: hookPath });
+                if (hook.sandbox?.enabled) {
+                    await this.registerSandboxedApiExtension(hook);
+                }
+                else {
+                    const hookPath = path.resolve(hook.path, hook.entrypoint);
+                    const hookInstance = await importFileUrl(hookPath, import.meta.url, {
+                        fresh: true,
+                    });
+                    const config = getModuleDefault(hookInstance);
+                    const unregisterFunctions = this.registerHook(config, hook.name);
+                    this.unregisterFunctionMap.set(hook.name, async () => {
+                        await Promise.all(unregisterFunctions.map((fn) => fn()));
+                        deleteFromRequireCache(hookPath);
+                    });
+                }
             }
             catch (error) {
                 logger.warn(`Couldn't register hook "${hook.name}"`);
@@ -248,15 +374,32 @@ export class ExtensionManager {
             }
         }
     }
+    /**
+     * Import the endpoint module code for all endpoint extensions, and register them individually through
+     * registerEndpoint
+     */
     async registerEndpoints() {
         const endpoints = this.extensions.filter((extension) => extension.type === 'endpoint');
         for (const endpoint of endpoints) {
+            const { enabled } = this.extensionsSettings.find(({ name }) => name === endpoint.name) ?? { enabled: false };
+            if (!enabled)
+                continue;
             try {
-                const endpointPath = path.resolve(endpoint.path, endpoint.entrypoint);
-                const endpointInstance = await import(`./${pathToRelativeUrl(endpointPath, __dirname)}?t=${Date.now()}`);
-                const config = getModuleDefault(endpointInstance);
-                this.registerEndpoint(config, endpoint.name);
-                this.apiExtensions.push({ path: endpointPath });
+                if (endpoint.sandbox?.enabled) {
+                    await this.registerSandboxedApiExtension(endpoint);
+                }
+                else {
+                    const endpointPath = path.resolve(endpoint.path, endpoint.entrypoint);
+                    const endpointInstance = await importFileUrl(endpointPath, import.meta.url, {
+                        fresh: true,
+                    });
+                    const config = getModuleDefault(endpointInstance);
+                    const unregister = this.registerEndpoint(config, endpoint.name);
+                    this.unregisterFunctionMap.set(endpoint.name, async () => {
+                        await unregister();
+                        deleteFromRequireCache(endpointPath);
+                    });
+                }
             }
             catch (error) {
                 logger.warn(`Couldn't register endpoint "${endpoint.name}"`);
@@ -264,6 +407,10 @@ export class ExtensionManager {
             }
         }
     }
+    /**
+     * Import the operation module code for all operation extensions, and register them individually through
+     * registerOperation
+     */
     async registerOperations() {
         const internalOperations = await readdir(path.join(__dirname, '..', 'operations'));
         for (const operation of internalOperations) {
@@ -273,12 +420,25 @@ export class ExtensionManager {
         }
         const operations = this.extensions.filter((extension) => extension.type === 'operation');
         for (const operation of operations) {
+            const { enabled } = this.extensionsSettings.find(({ name }) => name === operation.name) ?? { enabled: false };
+            if (!enabled)
+                continue;
             try {
-                const operationPath = path.resolve(operation.path, operation.entrypoint.api);
-                const operationInstance = await import(`./${pathToRelativeUrl(operationPath, __dirname)}?t=${Date.now()}`);
-                const config = getModuleDefault(operationInstance);
-                this.registerOperation(config);
-                this.apiExtensions.push({ path: operationPath });
+                if (operation.sandbox?.enabled) {
+                    await this.registerSandboxedApiExtension(operation);
+                }
+                else {
+                    const operationPath = path.resolve(operation.path, operation.entrypoint.api);
+                    const operationInstance = await importFileUrl(operationPath, import.meta.url, {
+                        fresh: true,
+                    });
+                    const config = getModuleDefault(operationInstance);
+                    const unregister = this.registerOperation(config);
+                    this.unregisterFunctionMap.set(operation.name, async () => {
+                        await unregister();
+                        deleteFromRequireCache(operationPath);
+                    });
+                }
             }
             catch (error) {
                 logger.warn(`Couldn't register operation "${operation.name}"`);
@@ -286,23 +446,36 @@ export class ExtensionManager {
             }
         }
     }
+    /**
+     * Import the module code for all hook, endpoint, and operation extensions registered within a
+     * bundle, and register them with their respective registration function
+     */
     async registerBundles() {
         const bundles = this.extensions.filter((extension) => extension.type === 'bundle');
         for (const bundle of bundles) {
             try {
                 const bundlePath = path.resolve(bundle.path, bundle.entrypoint.api);
-                const bundleInstances = await import(`./${pathToRelativeUrl(bundlePath, __dirname)}?t=${Date.now()}`);
+                const bundleInstances = await importFileUrl(bundlePath, import.meta.url, {
+                    fresh: true,
+                });
                 const configs = getModuleDefault(bundleInstances);
+                const unregisterFunctions = [];
                 for (const { config, name } of configs.hooks) {
-                    this.registerHook(config, name);
+                    const unregisters = this.registerHook(config, name);
+                    unregisterFunctions.push(...unregisters);
                 }
                 for (const { config, name } of configs.endpoints) {
-                    this.registerEndpoint(config, name);
+                    const unregister = this.registerEndpoint(config, name);
+                    unregisterFunctions.push(unregister);
                 }
                 for (const { config } of configs.operations) {
-                    this.registerOperation(config);
+                    const unregister = this.registerOperation(config);
+                    unregisterFunctions.push(unregister);
                 }
-                this.apiExtensions.push({ path: bundlePath });
+                this.unregisterFunctionMap.set(bundle.name, async () => {
+                    await Promise.all(unregisterFunctions.map((fn) => fn()));
+                    deleteFromRequireCache(bundlePath);
+                });
             }
             catch (error) {
                 logger.warn(`Couldn't register bundle "${bundle.name}"`);
@@ -310,31 +483,29 @@ export class ExtensionManager {
             }
         }
     }
-    registerHook(register, name) {
+    /**
+     * Register a single hook
+     */
+    registerHook(hookRegistrationCallback, name) {
         let scheduleIndex = 0;
-        const registerFunctions = {
+        const unregisterFunctions = [];
+        const hookRegistrationContext = {
             filter: (event, handler) => {
                 emitter.onFilter(event, handler);
-                this.hookEvents.push({
-                    type: 'filter',
-                    name: event,
-                    handler,
+                unregisterFunctions.push(() => {
+                    emitter.offFilter(event, handler);
                 });
             },
             action: (event, handler) => {
                 emitter.onAction(event, handler);
-                this.hookEvents.push({
-                    type: 'action',
-                    name: event,
-                    handler,
+                unregisterFunctions.push(() => {
+                    emitter.offAction(event, handler);
                 });
             },
             init: (event, handler) => {
                 emitter.onInit(event, handler);
-                this.hookEvents.push({
-                    type: 'init',
-                    name: event,
-                    handler,
+                unregisterFunctions.push(() => {
+                    emitter.offInit(name, handler);
                 });
             },
             schedule: (cron, handler) => {
@@ -350,9 +521,8 @@ export class ExtensionManager {
                         }
                     });
                     scheduleIndex++;
-                    this.hookEvents.push({
-                        type: 'schedule',
-                        job,
+                    unregisterFunctions.push(async () => {
+                        await job.stop();
                     });
                 }
                 else {
@@ -361,69 +531,74 @@ export class ExtensionManager {
             },
             embed: (position, code) => {
                 const content = typeof code === 'function' ? code() : code;
-                if (content.trim().length === 0) {
-                    logger.warn(`Couldn't register embed hook. Provided code is empty!`);
-                    return;
-                }
-                if (position === 'head') {
-                    this.hookEmbedsHead.push(content);
+                if (content.trim().length !== 0) {
+                    if (position === 'head') {
+                        const index = this.hookEmbedsHead.length;
+                        this.hookEmbedsHead.push(content);
+                        unregisterFunctions.push(() => {
+                            this.hookEmbedsHead.splice(index, 1);
+                        });
+                    }
+                    else {
+                        const index = this.hookEmbedsBody.length;
+                        this.hookEmbedsBody.push(content);
+                        unregisterFunctions.push(() => {
+                            this.hookEmbedsBody.splice(index, 1);
+                        });
+                    }
                 }
-                if (position === 'body') {
-                    this.hookEmbedsBody.push(content);
+                else {
+                    logger.warn(`Couldn't register embed hook. Provided code is empty!`);
                 }
             },
         };
-        register(registerFunctions, {
+        hookRegistrationCallback(hookRegistrationContext, {
             services,
             env,
             database: getDatabase(),
-            emitter: this.apiEmitter,
+            emitter: this.localEmitter,
             logger,
             getSchema,
         });
+        return unregisterFunctions;
     }
+    /**
+     * Register an individual endpoint
+     */
     registerEndpoint(config, name) {
-        const register = typeof config === 'function' ? config : config.handler;
+        const endpointRegistrationCallback = typeof config === 'function' ? config : config.handler;
         const routeName = typeof config === 'function' ? name : config.id;
         const scopedRouter = express.Router();
         this.endpointRouter.use(`/${routeName}`, scopedRouter);
-        register(scopedRouter, {
+        endpointRegistrationCallback(scopedRouter, {
             services,
             env,
             database: getDatabase(),
-            emitter: this.apiEmitter,
+            emitter: this.localEmitter,
             logger,
             getSchema,
         });
+        const unregisterFunction = () => {
+            this.endpointRouter.stack = this.endpointRouter.stack.filter((layer) => scopedRouter !== layer.handle);
+        };
+        return unregisterFunction;
     }
+    /**
+     * Register an individual operation
+     */
     registerOperation(config) {
         const flowManager = getFlowManager();
         flowManager.addOperation(config.id, config.handler);
+        const unregisterFunction = () => {
+            flowManager.removeOperation(config.id);
+        };
+        return unregisterFunction;
     }
+    /**
+     * Remove the registration for all API extensions
+     */
     async unregisterApiExtensions() {
-        for (const event of this.hookEvents) {
-            switch (event.type) {
-                case 'filter':
-                    emitter.offFilter(event.name, event.handler);
-                    break;
-                case 'action':
-                    emitter.offAction(event.name, event.handler);
-                    break;
-                case 'init':
-                    emitter.offInit(event.name, event.handler);
-                    break;
-                case 'schedule':
-                    await event.job.stop();
-                    break;
-            }
-        }
-        this.hookEvents = [];
-        this.endpointRouter.stack = [];
-        const flowManager = getFlowManager();
-        flowManager.clearOperations();
-        for (const apiExtension of this.apiExtensions) {
-            delete require.cache[require.resolve(apiExtension.path)];
-        }
-        this.apiExtensions = [];
+        const unregisterFunctions = Array.from(this.unregisterFunctionMap.values());
+        await Promise.all(unregisterFunctions.map((fn) => fn()));
     }
 }

package/dist/extensions/types.d.ts CHANGED Viewed

@@ -13,11 +13,7 @@ export type BundleConfig = {
         config: OperationApiConfig;
     }[];
 };
-export type AppExtensions = string | null;
-export type ApiExtensions = {
-    path: string;
-}[];
-export interface Options {
+export interface ExtensionManagerOptions {
     schedule: boolean;
     watch: boolean;
 }