@directus/api 17.1.0 → 18.1.0

package/dist/extensions/lib/installation/manager.js

@@ -0,0 +1,90 @@
+import { useEnv } from '@directus/env';
+import { ServiceUnavailableError } from '@directus/errors';
+import { EXTENSION_PKG_KEY, ExtensionManifest } from '@directus/extensions';
+import { download } from '@directus/extensions-registry';
+import DriverLocal from '@directus/storage-driver-local';
+import { move, remove } from 'fs-extra';
+import { mkdir, readFile, rm } from 'node:fs/promises';
+import { Readable } from 'node:stream';
+import Queue from 'p-queue';
+import { join } from 'path';
+import tar from 'tar';
+import { useLogger } from '../../../logger.js';
+import { getStorage } from '../../../storage/index.js';
+import { getExtensionsPath } from '../get-extensions-path.js';
+const env = useEnv();
+export class InstallationManager {
+    extensionPath = getExtensionsPath();
+    async install(versionId) {
+        const logger = useLogger();
+        const tempDir = join(env['TEMP_PATH'], 'marketplace', versionId);
+        const tmpStorage = new DriverLocal({ root: tempDir });
+        try {
+            await mkdir(tempDir, { recursive: true });
+            const options = {};
+            if (env['MARKETPLACE_REGISTRY'] && typeof env['MARKETPLACE_REGISTRY'] === 'string') {
+                options.registry = env['MARKETPLACE_REGISTRY'];
+            }
+            const tarReadableStream = await download(versionId, env['MARKETPLACE_TRUST'] === 'sandbox', options);
+            if (!tarReadableStream) {
+                throw new Error(`No readable stream returned from download`);
+            }
+            const tarStream = Readable.fromWeb(tarReadableStream);
+            const tarPath = join(tempDir, `bin.tar.tgz`);
+            await tmpStorage.write('bin.tar.tgz', tarStream);
+            /**
+             * NPM modules that are packed are always tarballed in a folder called "package"
+             */
+            const extractedPath = 'package';
+            await tar.extract({
+                file: tarPath,
+                cwd: tempDir,
+            });
+            const packageFile = JSON.parse(await readFile(join(tempDir, extractedPath, 'package.json'), { encoding: 'utf-8' }));
+            const extensionManifest = await ExtensionManifest.parseAsync(packageFile);
+            if (!extensionManifest[EXTENSION_PKG_KEY]?.type) {
+                throw new Error(`Extension type not found in package.json`);
+            }
+            if (env['EXTENSIONS_LOCATION']) {
+                // Upload the extension into the configured extensions location
+                const storage = await getStorage();
+                const remoteDisk = storage.location(env['EXTENSIONS_LOCATION']);
+                const queue = new Queue({ concurrency: 1000 });
+                for await (const filepath of tmpStorage.list(extractedPath)) {
+                    const readStream = await tmpStorage.read(filepath);
+                    const remotePath = join(env['EXTENSIONS_PATH'], '.registry', versionId, filepath.substring(extractedPath.length));
+                    queue.add(() => remoteDisk.write(remotePath, readStream));
+                }
+                await queue.onIdle();
+            }
+            else {
+                // No custom location, so save to regular local extensions folder
+                const dest = join(this.extensionPath, '.registry', versionId);
+                await move(join(tempDir, extractedPath), dest, { overwrite: true });
+            }
+        }
+        catch (err) {
+            logger.warn(err);
+            throw new ServiceUnavailableError({ service: 'marketplace', reason: 'Could not download and extract the extension' }, { cause: err });
+        }
+        finally {
+            await rm(tempDir, { recursive: true });
+        }
+    }
+    async uninstall(folder) {
+        if (env['EXTENSIONS_LOCATION']) {
+            const storage = await getStorage();
+            const remoteDisk = storage.location(env['EXTENSIONS_LOCATION']);
+            const queue = new Queue({ concurrency: 1000 });
+            const prefix = join(env['EXTENSIONS_PATH'], '.registry', folder);
+            for await (const filepath of remoteDisk.list(prefix)) {
+                queue.add(() => remoteDisk.delete(filepath));
+            }
+            await queue.onIdle();
+        }
+        else {
+            const path = join(this.extensionPath, '.registry', folder);
+            await remove(path);
+        }
+    }
+}

package/dist/extensions/lib/sandbox/generate-api-extensions-sandbox-entrypoint.d.ts

@@ -16,7 +16,7 @@ export declare function generateApiExtensionsSandboxEntrypoint(type: ApiExtensio
     unregisterFunction: () => Promise<void>;
 } | {
     code: string;
-    hostFunctions: ((path: import("isolated-vm").Reference<string>, method: import("isolated-vm").Reference<"GET" | "POST" | "DELETE" | "PATCH" | "PUT">, cb: import("isolated-vm").Reference<(req: {
+    hostFunctions: ((path: import("isolated-vm").Reference<string>, method: import("isolated-vm").Reference<"GET" | "POST" | "DELETE" | "PUT" | "PATCH">, cb: import("isolated-vm").Reference<(req: {
         url: string;
         headers: import("http").IncomingHttpHeaders;
         body: string;

package/dist/extensions/lib/sync-extensions.js

@@ -1,9 +1,8 @@
 import { useEnv } from '@directus/env';
-import { NESTED_EXTENSION_TYPES } from '@directus/extensions';
-import { ensureExtensionDirs } from '@directus/extensions/node';
+import { exists } from 'fs-extra';
 import mid from 'node-machine-id';
 import { createWriteStream } from 'node:fs';
-import { mkdir } from 'node:fs/promises';
+import { mkdir, rm } from 'node:fs/promises';
 import { dirname, join, relative, resolve, sep } from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import Queue from 'p-queue';
@@ -16,10 +15,7 @@ export const syncExtensions = async () => {
     const env = useEnv();
     const logger = useLogger();
     const extensionsPath = getExtensionsPath();
-    if (!env['EXTENSIONS_LOCATION']) {
-        // Safe to run with multiple instances since dirs are created with `recursive: true`
-        return ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
-    }
+    const storageExtensionsPath = env['EXTENSIONS_PATH'];
     const messenger = useBus();
     const isPrimaryProcess = String(process.env['NODE_APP_INSTANCE']) === '0' || process.env['NODE_APP_INSTANCE'] === undefined;
     const id = await mid.machineId();
@@ -36,6 +32,12 @@ export const syncExtensions = async () => {
             messenger.subscribe(message, () => resolve());
         });
     }
+    if (await exists(extensionsPath)) {
+        // In case the FS still contains the cached extensions from a previous invocation. We have to
+        // clear them out to ensure the remote extensions folder remains the source of truth for all
+        // extensions that are loaded.
+        await rm(extensionsPath, { recursive: true, force: true });
+    }
     // Ensure that the local extensions cache path exists
     await mkdir(extensionsPath, { recursive: true });
     await setSyncStatus(SyncStatus.SYNCING);
@@ -44,18 +46,17 @@ export const syncExtensions = async () => {
     const disk = storage.location(env['EXTENSIONS_LOCATION']);
     // Make sure we don't overload the file handles
     const queue = new Queue({ concurrency: 1000 });
-    for await (const filepath of disk.list(env['EXTENSIONS_PATH'])) {
+    for await (const filepath of disk.list(storageExtensionsPath)) {
         const readStream = await disk.read(filepath);
         // We want files to be stored in the root of `$TEMP_PATH/extensions`, so gotta remove the
         // extensions path on disk from the start of the file path
-        const destPath = join(extensionsPath, relative(resolve(sep, env['EXTENSIONS_PATH']), resolve(sep, filepath)));
+        const destPath = join(extensionsPath, relative(resolve(sep, storageExtensionsPath), resolve(sep, filepath)));
         // Ensure that the directory path exists
         await mkdir(dirname(destPath), { recursive: true });
         const writeStream = createWriteStream(destPath);
         queue.add(() => pipeline(readStream, writeStream));
     }
     await queue.onIdle();
-    await ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
     await setSyncStatus(SyncStatus.DONE);
     messenger.publish(message, { ready: true });
 };

package/dist/extensions/manager.d.ts

@@ -7,10 +7,9 @@ export declare class ExtensionManager {
      * Whether or not the extensions have been read from disk and registered into the system
      */
     private isLoaded;
-    /**
-     * All extensions that are loaded within the current process
-     */
-    private extensions;
+    private localExtensions;
+    private registryExtensions;
+    private moduleExtensions;
     /**
      * Settings for the extensions that are loaded within the current process
      */
@@ -57,6 +56,18 @@ export declare class ExtensionManager {
      * Optional file system watcher to auto-reload extensions when the local file system changes
      */
     private watcher;
+    /**
+     * installation manager responsible for installing extensions from registries
+     */
+    private installationManager;
+    private messenger;
+    /**
+     * channel to publish on registering extension from external registry
+     */
+    private reloadChannel;
+    private processId;
+    get extensions(): Extension[];
+    getExtension(source: string, folder: string): Extension | undefined;
     /**
      * Load and register all extensions
      *
@@ -65,6 +76,11 @@ export declare class ExtensionManager {
      * @param {boolean} options.watch - Whether or not to watch the local extensions folder for changes
      */
     initialize(options?: Partial<ExtensionManagerOptions>): Promise<void>;
+    /**
+     * Installs an external extension from registry
+     */
+    install(versionId: string): Promise<void>;
+    uninstall(folder: string): Promise<void>;
     /**
      * Load all extensions from disk and register them in their respective places
      */
@@ -76,7 +92,7 @@ export declare class ExtensionManager {
     /**
      * Reload all the extensions. Will unload if extensions have already been loaded
      */
-    reload(): void;
+    reload(): Promise<unknown>;
     /**
      * Return the previously generated app extensions bundle
      */
@@ -96,10 +112,6 @@ export declare class ExtensionManager {
         head: string;
         body: string;
     };
-    /**
-     * Allow reading the installed extensions
-     */
-    getExtensions(): Extension[];
     /**
      * Start the chokidar watcher for extensions on the local filesystem
      */
@@ -119,26 +131,16 @@ export declare class ExtensionManager {
      */
     private generateExtensionBundle;
     private registerSandboxedApiExtension;
-    /**
-     * Import the hook module code for all hook extensions, and register them individually through
-     * registerHook
-     */
-    private registerHooks;
-    /**
-     * Import the endpoint module code for all endpoint extensions, and register them individually through
-     * registerEndpoint
-     */
-    private registerEndpoints;
+    private registerApiExtensions;
+    private registerHookExtension;
+    private registerEndpointExtension;
+    private registerOperationExtension;
+    private registerBundleExtension;
     /**
      * Import the operation module code for all operation extensions, and register them individually through
      * registerOperation
      */
-    private registerOperations;
-    /**
-     * Import the module code for all hook, endpoint, and operation extensions registered within a
-     * bundle, and register them with their respective registration function
-     */
-    private registerBundles;
+    private registerInternalOperations;
     /**
      * Register a single hook
      */