@jupyterlite/xeus 4.0.0-a0 → 4.0.0-a10

package/lib/worker.js

@@ -2,28 +2,8 @@
 // Copyright (c) JupyterLite Contributors
 // Distributed under the terms of the Modified BSD License.
 import { URLExt } from '@jupyterlab/coreutils';
-import { bootstrapEmpackPackedEnvironment, bootstrapPython, getPythonVersion, loadShareLibs, waitRunDependencies } from '@emscripten-forge/mambajs';
-globalThis.Module = {};
-// when a toplevel cell uses an await, the cell is implicitly
-// wrapped in a async function. Since the webloop - eventloop
-// implementation does not support `eventloop.run_until_complete(f)`
-// we need to convert the toplevel future in a javascript Promise
-// this `toplevel` promise is then awaited before we
-// execute the next cell. After the promise is awaited we need
-// to do some cleanup and delete the python proxy
-// (ie a js-wrapped python object) to avoid memory leaks
-globalThis.toplevel_promise = null;
-globalThis.toplevel_promise_py_proxy = null;
-let resolveInputReply;
-let kernelReady;
-let rawXKernel;
-let rawXServer;
-async function get_stdin() {
-    const replyPromise = new Promise(resolve => {
-        resolveInputReply = resolve;
-    });
-    return replyPromise;
-}
+import { bootstrapEmpackPackedEnvironment, bootstrapPython, getPythonVersion, loadShareLibs, solve, installPackagesToEmscriptenFS, removePackagesFromEmscriptenFS, showPackagesList } from '@emscripten-forge/mambajs';
+import { XeusRemoteKernelBase } from './worker.base';
 async function fetchJson(url) {
     const response = await fetch(url);
     if (!response.ok) {
@@ -32,131 +12,190 @@ async function fetchJson(url) {
     const json = await response.json();
     return json;
 }
-self.get_stdin = get_stdin;
-globalThis.ready = new Promise(resolve => {
-    kernelReady = resolve;
-});
-export class XeusRemoteKernel {
-    constructor(options = {}) {
-        this._sendWorkerMessage = () => { };
-    }
-    async ready() {
-        return await globalThis.ready;
-    }
-    async cd(path) {
-        if (!path || !globalThis.Module.FS) {
-            return;
-        }
-        globalThis.Module.FS.chdir(path);
-    }
-    async isDir(path) {
-        try {
-            const lookup = globalThis.Module.FS.lookupPath(path);
-            return globalThis.Module.FS.isDir(lookup.node.mode);
-        }
-        catch (e) {
-            return false;
-        }
+/**
+ * A worker kernel that is backed by an empack environment
+ */
+export class EmpackedXeusRemoteKernel extends XeusRemoteKernelBase {
+    constructor() {
+        super(...arguments);
+        this._prefix = '';
+        this._pkgRootUrl = '';
+        this._installedPackages = {};
+        this._paths = {};
     }
-    async processMessage(event) {
-        const msg_type = event.msg.header.msg_type;
-        await globalThis.ready;
-        if (globalThis.toplevel_promise !== null &&
-            globalThis.toplevel_promise_py_proxy !== null) {
-            await globalThis.toplevel_promise;
-            globalThis.toplevel_promise_py_proxy.delete();
-            globalThis.toplevel_promise_py_proxy = null;
-            globalThis.toplevel_promise = null;
-        }
-        if (msg_type === 'input_reply') {
-            resolveInputReply(event.msg);
-        }
-        else {
-            rawXServer.notify_listener(event.msg);
-        }
-    }
-    async initialize(options) {
-        const { baseUrl, kernelSpec, empackEnvMetaLink } = options;
+    async initializeModule(options) {
+        const { baseUrl, kernelSpec } = options;
         // location of the kernel binary on the server
-        const binary_js = URLExt.join(baseUrl, kernelSpec.argv[0]);
-        const binary_wasm = binary_js.replace('.js', '.wasm');
-        const binary_data = binary_js.replace('.js', '.data');
-        const kernel_root_url = URLExt.join(baseUrl, 'xeus', 'kernels', kernelSpec.dir);
+        const binaryJS = URLExt.join(baseUrl, kernelSpec.argv[0]);
+        const binaryWASM = binaryJS.replace('.js', '.wasm');
+        const binaryDATA = binaryJS.replace('.js', '.data');
+        const kernelRootUrl = URLExt.join(baseUrl, 'xeus', kernelSpec.envName);
         const sharedLibs = kernelSpec.metadata && kernelSpec.metadata.shared
             ? kernelSpec.metadata.shared
             : {};
-        importScripts(binary_js);
-        globalThis.Module = await createXeusModule({
+        importScripts(binaryJS);
+        return {
             locateFile: (file) => {
                 if (file in sharedLibs) {
-                    return URLExt.join(kernel_root_url, file);
+                    return URLExt.join(kernelRootUrl, kernelSpec.name, file);
                 }
                 if (file.endsWith('.wasm')) {
-                    return binary_wasm;
+                    return binaryWASM;
                 }
                 else if (file.endsWith('.data')) {
                     // Handle the .data file if it exists
-                    return binary_data;
+                    return binaryDATA;
                 }
                 return file;
             }
+        };
+    }
+    async initializeFileSystem(options) {
+        const { baseUrl, kernelSpec, empackEnvMetaLink } = options;
+        if (this.Module.FS === undefined ||
+            this.Module.loadDynamicLibrary === undefined) {
+            throw new Error('Cannot load kernel without a valid FS');
+        }
+        // location of the kernel binary on the server
+        const kernelRootUrl = URLExt.join(baseUrl, 'xeus', 'kernels', kernelSpec.dir);
+        const empackEnvMetaLocation = empackEnvMetaLink || kernelRootUrl;
+        const packagesJsonUrl = `${empackEnvMetaLocation}/empack_env_meta.json`;
+        this._pkgRootUrl = URLExt.join(baseUrl, `xeus/${kernelSpec.envName}/kernel_packages`);
+        const empackEnvMeta = (await fetchJson(packagesJsonUrl));
+        // Initialize installed packages from empack env meta
+        this._installedPackages = {};
+        empackEnvMeta.packages.map(pkg => {
+            this._installedPackages[pkg.filename] = {
+                name: pkg.name,
+                version: pkg.version,
+                repo_url: pkg.channel ? pkg.channel : '',
+                url: pkg.url ? pkg.url : '',
+                repo_name: pkg.channel ? pkg.channel : '',
+                build_string: pkg.build
+            };
         });
-        try {
-            await waitRunDependencies(globalThis.Module);
-            if (globalThis.Module.FS !== undefined &&
-                globalThis.Module.loadDynamicLibrary !== undefined) {
-                const empackEnvMetaLocation = empackEnvMetaLink || kernel_root_url;
-                const packagesJsonUrl = `${empackEnvMetaLocation}/empack_env_meta.json`;
-                const pkgRootUrl = URLExt.join(baseUrl, `xeus/kernels/${kernelSpec.name}/kernel_packages`);
-                const empackEnvMeta = (await fetchJson(packagesJsonUrl));
-                const sharedLibs = await bootstrapEmpackPackedEnvironment({
-                    empackEnvMeta,
-                    pkgRootUrl,
-                    Module: globalThis.Module,
-                    verbose: false
-                });
-                // Bootstrap Python, if it's xeus-python
-                if (kernelSpec.name === 'xpython') {
-                    const pythonVersion = getPythonVersion(empackEnvMeta.packages);
-                    if (!pythonVersion) {
-                        throw new Error('Failed to load Python!');
-                    }
-                    await bootstrapPython({
-                        prefix: empackEnvMeta.prefix,
-                        pythonVersion: pythonVersion,
-                        Module: globalThis.Module
-                    });
-                }
-                // Load shared libs
-                await loadShareLibs({
-                    sharedLibs,
-                    prefix: empackEnvMeta.prefix,
-                    Module: globalThis.Module
-                });
-            }
-            rawXKernel = new globalThis.Module.xkernel();
-            rawXServer = rawXKernel.get_server();
-            if (!rawXServer) {
-                console.error('Failed to start kernel!');
+        this._pythonVersion = getPythonVersion(empackEnvMeta.packages);
+        this._prefix = empackEnvMeta.prefix;
+        const bootstrapped = await bootstrapEmpackPackedEnvironment({
+            empackEnvMeta,
+            pkgRootUrl: this._pkgRootUrl,
+            Module: this.Module,
+            logger: this.logger,
+            pythonVersion: this._pythonVersion
+        });
+        this._paths = bootstrapped.paths;
+        this._untarjs = bootstrapped.untarjs;
+        this._sharedLibs = bootstrapped.sharedLibs;
+    }
+    /**
+     * Initialize the interpreter if needed
+     * @param options
+     */
+    async initializeInterpreter(options) {
+        // Bootstrap Python, if it's xeus-python
+        if (options.kernelSpec.name === 'xpython') {
+            if (!this._pythonVersion) {
+                throw new Error('Python is not installed, cannot start Python!');
             }
-            rawXKernel.start();
+            this.logger.log('Starting Python');
+            await bootstrapPython({
+                prefix: this._prefix,
+                pythonVersion: this._pythonVersion,
+                Module: this.Module
+            });
         }
-        catch (e) {
-            if (typeof e === 'number') {
-                const msg = globalThis.Module.get_exception_message(e);
-                throw new Error(msg);
+        // Load shared libs
+        await loadShareLibs({
+            sharedLibs: this._sharedLibs,
+            prefix: this._prefix,
+            Module: this.Module,
+            logger: this.logger
+        });
+    }
+    async install(channels, specs, pipSpecs) {
+        var _a, _b;
+        if (specs.length || pipSpecs.length) {
+            try {
+                this.logger.log(`Collecting ${(_a = (specs || pipSpecs)) === null || _a === void 0 ? void 0 : _a.join(',')}`);
+                const newPackages = await solve({
+                    ymlOrSpecs: specs,
+                    installedPackages: this._installedPackages,
+                    pipSpecs,
+                    channels,
+                    logger: this.logger
+                });
+                await this._reloadPackagesInFS({
+                    ...newPackages.condaPackages,
+                    ...newPackages.pipPackages
+                });
             }
-            else {
-                throw e;
+            catch (error) {
+                (_b = this.logger) === null || _b === void 0 ? void 0 : _b.error(error.stack);
             }
         }
-        kernelReady(1);
     }
     /**
-     * Register the callback function to send messages from the worker back to the main thread.
-     * @param callback the callback to register
+     * Implements dynamic installation of packages
      */
-    registerCallback(callback) {
-        this._sendWorkerMessage = callback;
+    listInstalledPackages() {
+        showPackagesList(this._installedPackages, this.logger);
+        return Promise.resolve();
+    }
+    async _reloadPackagesInFS(newInstalledPackages) {
+        const removedPackages = {};
+        const newPackages = {};
+        // First create structures we can quickly inspect
+        const newInstalledPackagesMap = {};
+        for (const newInstalledPkg of Object.values(newInstalledPackages)) {
+            newInstalledPackagesMap[newInstalledPkg.name] = newInstalledPkg;
+        }
+        const oldInstalledPackagesMap = {};
+        for (const oldInstalledPkg of Object.values(this._installedPackages)) {
+            oldInstalledPackagesMap[oldInstalledPkg.name] = oldInstalledPkg;
+        }
+        // Compare old installed packages with new ones
+        for (const filename of Object.keys(this._installedPackages)) {
+            const installedPkg = this._installedPackages[filename];
+            // Exact same build of the package already installed
+            if (installedPkg.name in newInstalledPackagesMap &&
+                installedPkg.build_string ===
+                    newInstalledPackagesMap[installedPkg.name].build_string) {
+                continue;
+            }
+            removedPackages[filename] = installedPkg;
+        }
+        // Compare new installed packages with old ones
+        for (const filename of Object.keys(newInstalledPackages)) {
+            const newPkg = newInstalledPackages[filename];
+            // Exact same build of the package already installed
+            if (newPkg.name in oldInstalledPackagesMap &&
+                newPkg.build_string ===
+                    oldInstalledPackagesMap[newPkg.name].build_string) {
+                continue;
+            }
+            newPackages[filename] = newPkg;
+        }
+        await removePackagesFromEmscriptenFS({
+            removedPackages,
+            Module: this.Module,
+            paths: this._paths,
+            logger: this.logger
+        });
+        const { sharedLibs, paths } = await installPackagesToEmscriptenFS({
+            packages: newPackages,
+            pkgRootUrl: this._pkgRootUrl,
+            pythonVersion: this._pythonVersion,
+            Module: this.Module,
+            untarjs: this._untarjs,
+            logger: this.logger
+        });
+        this._paths = { ...this._paths, ...paths };
+        await loadShareLibs({
+            sharedLibs,
+            prefix: this._prefix,
+            Module: this.Module,
+            logger: this.logger
+        });
+        this._installedPackages = newInstalledPackages;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@jupyterlite/xeus",
-    "version": "4.0.0-a0",
+    "version": "4.0.0-a10",
     "description": "JupyterLite Xeus kernels",
     "keywords": [
         "jupyter",
@@ -36,19 +36,18 @@
         "watch:src": "tsc -w --sourceMap"
     },
     "dependencies": {
-        "@emscripten-forge/mambajs": "^0.3.1",
-        "@jupyterlab/coreutils": "^6",
-        "@jupyterlab/services": "^7.3.4",
-        "@jupyterlite/contents": "^0.2.0 || ^0.3.0 || ^0.4.5 || ^0.5.1",
-        "@jupyterlite/kernel": "^0.2.0 || ^0.3.0 || ^0.4.5 || ^0.5.1",
-        "@jupyterlite/server": "^0.2.0 || ^0.3.0 || ^0.4.5 || ^0.5.1",
+        "@emscripten-forge/mambajs": "^0.12.0",
+        "@jupyterlab/coreutils": "^6.4.2",
+        "@jupyterlab/services": "^7.4.2",
+        "@jupyterlite/contents": "^0.6.0-beta.0",
+        "@jupyterlite/kernel": "^0.6.0-beta.0",
+        "@jupyterlite/server": "^0.6.0-beta.0",
         "@lumino/coreutils": "^2",
         "@lumino/signaling": "^2",
         "coincident": "^1.2.3",
         "comlink": "^4.4.1"
     },
     "devDependencies": {
-        "@jupyterlab/builder": "^4.1.0",
         "@types/json-schema": "^7.0.11",
         "@types/react": "^18.0.26",
         "@types/react-addons-linked-state-mixin": "^0.14.22",
@@ -56,7 +55,7 @@
         "rimraf": "^5.0.1",
         "source-map-loader": "^1.0.2",
         "ts-loader": "^9.2.6",
-        "typescript": "~5.0.2",
+        "typescript": "^5.5",
         "webpack": "^5.87.0",
         "yjs": "^13.5.0"
     },