@jupyterlite/xeus 4.0.0-a0 → 4.0.0-a10

package/lib/index.d.ts

@@ -1,5 +1,4 @@
-export * from './coincident.worker';
-export * from './comlink.worker';
 export * from './interfaces';
 export * from './web_worker_kernel';
+export * from './worker.base';
 export * from './worker';

package/lib/index.js

@@ -1,5 +1,4 @@
-export * from './coincident.worker';
-export * from './comlink.worker';
 export * from './interfaces';
 export * from './web_worker_kernel';
+export * from './worker.base';
 export * from './worker';

package/lib/interfaces.d.ts

@@ -1,6 +1,7 @@
 /**
  * Definitions for the Xeus kernel.
  */
+import type { KernelMessage } from '@jupyterlab/services';
 import { TDriveMethod, TDriveRequest, TDriveResponse } from '@jupyterlite/contents';
 import { IWorkerKernel } from '@jupyterlite/kernel';
 /**
@@ -21,15 +22,17 @@ export interface IXeusWorkerKernel extends IWorkerKernel {
      * @param msg
      */
     processMessage(msg: any): void;
+    /**
+     * Process stdin request, blocking until the reply is received.
+     * This is sync for the web worker, async for the UI thread.
+     * @param inputRequest
+     */
+    processStdinRequest(inputRequest: KernelMessage.IInputRequestMsg): KernelMessage.IInputReplyMsg;
     /**
      * Process worker message
      * @param msg
      */
     processWorkerMessage(msg: any): void;
-    /**
-     * Register a callback for handling messages from the worker.
-     */
-    registerCallback(callback: (msg: any) => void): void;
     /**
      * Whether the kernel is ready.
      * @returns a promise that resolves when the kernel is ready.
@@ -40,8 +43,9 @@ export interface IXeusWorkerKernel extends IWorkerKernel {
      * @param driveName The name of the drive
      * @param mountpoint The mountpoint of the drive
      * @param baseUrl The base URL of the server
+     * @param browsingContextId The current page id
      */
-    mount(driveName: string, mountpoint: string, baseUrl: string): Promise<void>;
+    mount(driveName: string, mountpoint: string, baseUrl: string, browsingContextId: string): Promise<void>;
     /**
      * Change the current working directory
      * @param path The path to change to
@@ -62,8 +66,10 @@ export declare namespace IXeusWorkerKernel {
      */
     interface IOptions extends IWorkerKernel.IOptions {
         baseUrl: string;
+        kernelId: string;
         kernelSpec: any;
         mountDrive: boolean;
         empackEnvMetaLink?: string;
+        browsingContextId: string;
     }
 }

package/lib/web_worker_kernel.d.ts

@@ -36,12 +36,17 @@ export declare class WebWorkerKernel implements IKernel {
      * Get the kernel location
      */
     get location(): string;
+    /**
+     * Send kernel message to the client
+     */
+    protected get sendMessage(): IKernel.SendMessage;
     /**
      * Process a message coming from the coincident web worker.
      *
      * @param msg The worker message to process.
      */
     private _processCoincidentWorkerMessage;
+    private _assignSession;
     /**
      * Process a message coming from the comlink web worker.
      *
@@ -85,6 +90,7 @@ export declare class WebWorkerKernel implements IKernel {
     private _worker;
     private _sendMessage;
     private _executeDelegate;
+    private _inputDelegate;
     private _parentHeader;
     private _parent;
     private _ready;
@@ -102,5 +108,6 @@ export declare namespace WebWorkerKernel {
         mountDrive: boolean;
         kernelSpec: any;
         empackEnvMetaLink?: string | undefined;
+        browsingContextId: string;
     }
 }

package/lib/web_worker_kernel.js

@@ -6,6 +6,7 @@ import { wrap } from 'comlink';
 import { Signal } from '@lumino/signaling';
 import { PromiseDelegate } from '@lumino/coreutils';
 import { PageConfig } from '@jupyterlab/coreutils';
+import { KernelMessage } from '@jupyterlab/services';
 import { DriveContentsProcessor } from '@jupyterlite/contents';
 export class WebWorkerKernel {
     /**
@@ -18,6 +19,7 @@ export class WebWorkerKernel {
         this._isDisposed = false;
         this._disposed = new Signal(this);
         this._executeDelegate = new PromiseDelegate();
+        this._inputDelegate = new PromiseDelegate();
         this._parentHeader = undefined;
         this._parent = undefined;
         this._ready = new PromiseDelegate();
@@ -74,6 +76,14 @@ export class WebWorkerKernel {
                 }
                 return await this._contentsProcessor.processDriveRequest(data);
             };
+            // Stdin request is synchronous from the web worker's point of view, blocking
+            // until the reply is received. From the UI thread's point of view it is async.
+            remote.processStdinRequest = async (inputRequest) => {
+                this._processCoincidentWorkerMessage({ data: inputRequest });
+                this._inputDelegate =
+                    new PromiseDelegate();
+                return await this._inputDelegate.promise;
+            };
         }
         else {
             this._worker.onmessage = e => {
@@ -83,10 +93,12 @@ export class WebWorkerKernel {
         }
         remote
             .initialize({
+            kernelId: this.id,
             kernelSpec: this._kernelSpec,
             baseUrl: PageConfig.getBaseUrl(),
             mountDrive: options.mountDrive,
-            empackEnvMetaLink: this._empackEnvMetaLink
+            empackEnvMetaLink: this._empackEnvMetaLink,
+            browsingContextId: options.browsingContextId
         })
             .then(this._ready.resolve.bind(this._ready));
         return remote;
@@ -97,11 +109,12 @@ export class WebWorkerKernel {
         await this._sendMessageToWorker(msg);
     }
     async _sendMessageToWorker(msg) {
-        if (msg.header.msg_type !== 'input_reply') {
-            this._executeDelegate = new PromiseDelegate();
+        if (msg.header.msg_type === 'input_reply') {
+            this._inputDelegate.resolve(msg);
         }
-        await this._remoteKernel.processMessage({ msg, parent: this.parent });
-        if (msg.header.msg_type !== 'input_reply') {
+        else {
+            this._executeDelegate = new PromiseDelegate();
+            await this._remoteKernel.processMessage({ msg, parent: this.parent });
             return await this._executeDelegate.promise;
         }
     }
@@ -123,6 +136,12 @@ export class WebWorkerKernel {
     get location() {
         return this._location;
     }
+    /**
+     * Send kernel message to the client
+     */
+    get sendMessage() {
+        return this._sendMessage;
+    }
     /**
      * Process a message coming from the coincident web worker.
      *
@@ -142,19 +161,60 @@ export class WebWorkerKernel {
             this._executeDelegate.resolve();
         }
     }
+    _assignSession(msg) {
+        var _a, _b, _c, _d;
+        msg.header.session = (_b = (_a = this._parentHeader) === null || _a === void 0 ? void 0 : _a.session) !== null && _b !== void 0 ? _b : '';
+        msg.session = (_d = (_c = this._parentHeader) === null || _c === void 0 ? void 0 : _c.session) !== null && _d !== void 0 ? _d : '';
+        return msg;
+    }
     /**
      * Process a message coming from the comlink web worker.
      *
      * @param msg The worker message to process.
      */
     _processComlinkWorkerMessage(msg) {
-        var _a, _b, _c, _d;
+        var _a, _b;
         if (!msg.header) {
-            return;
+            if (msg === null || msg === void 0 ? void 0 : msg._stream) {
+                const parentHeaderValue = this._parentHeader;
+                const { name, text } = msg._stream;
+                if (name === 'stderr') {
+                    const errorMessage = KernelMessage.createMessage({
+                        msgType: 'execute_reply',
+                        channel: 'shell',
+                        parentHeader: parentHeaderValue,
+                        session: (_a = parentHeaderValue === null || parentHeaderValue === void 0 ? void 0 : parentHeaderValue.session) !== null && _a !== void 0 ? _a : '',
+                        content: {
+                            execution_count: msg._stream.executionCount,
+                            status: 'error',
+                            ename: msg._stream.ename,
+                            evalue: msg._stream.evalue,
+                            traceback: msg._stream.traceback.join('')
+                        }
+                    });
+                    msg = this._assignSession(errorMessage);
+                    this._sendMessage(msg);
+                }
+                const message = KernelMessage.createMessage({
+                    channel: 'iopub',
+                    msgType: 'stream',
+                    session: (_b = parentHeaderValue === null || parentHeaderValue === void 0 ? void 0 : parentHeaderValue.session) !== null && _b !== void 0 ? _b : '',
+                    parentHeader: parentHeaderValue,
+                    content: {
+                        name,
+                        text
+                    }
+                });
+                msg = this._assignSession(message);
+                this._sendMessage(msg);
+            }
+            else {
+                return;
+            }
+        }
+        else {
+            this._sendMessage(this._assignSession(msg));
         }
-        msg.header.session = (_b = (_a = this._parentHeader) === null || _a === void 0 ? void 0 : _a.session) !== null && _b !== void 0 ? _b : '';
-        msg.session = (_d = (_c = this._parentHeader) === null || _c === void 0 ? void 0 : _c.session) !== null && _d !== void 0 ? _d : '';
-        this._sendMessage(msg);
         // resolve promise
         if (msg.header.msg_type === 'status' &&
             msg.content.execution_state === 'idle') {
@@ -217,12 +277,12 @@ export class WebWorkerKernel {
             localPath = options.location;
         }
         await this._remoteKernel.ready();
-        await this._remoteKernel.mount(driveName, '/drive', PageConfig.getBaseUrl());
+        await this._remoteKernel.mount(driveName, '/drive', PageConfig.getBaseUrl(), options.browsingContextId);
         if (await this._remoteKernel.isDir('/files')) {
             await this._remoteKernel.cd('/files');
         }
         else {
-            await this._remoteKernel.cd(localPath);
+            await this._remoteKernel.cd(`/drive/${localPath}`);
         }
     }
 }

package/lib/worker.base.d.ts

@@ -0,0 +1,76 @@
+import { IXeusWorkerKernel } from './interfaces';
+import { ILogger } from '@emscripten-forge/mambajs';
+export declare class XeusWorkerLoggerBase implements ILogger {
+    constructor(kernelId: string);
+    log(...msg: any[]): void;
+    warn(...msg: any[]): void;
+    error(...msg: any[]): void;
+    private _id;
+    private _channel;
+    executionCount: number;
+}
+/**
+ * The base class for the worker kernel
+ *
+ * Meant to be extended in order to load kernels from other sources, implement custom magics etc.
+ */
+export declare abstract class XeusRemoteKernelBase {
+    constructor(options?: XeusRemoteKernelBase.IOptions);
+    ready(): Promise<void>;
+    cd(path: string): Promise<void>;
+    isDir(path: string): Promise<boolean>;
+    processMessage(event: any): Promise<void>;
+    protected get Module(): any;
+    protected set Module(value: any);
+    protected get logger(): XeusWorkerLoggerBase;
+    initialize(options: IXeusWorkerKernel.IOptions): Promise<void>;
+    protected initializeLogger(options: IXeusWorkerKernel.IOptions): XeusWorkerLoggerBase;
+    /**
+     * Initialize the emscripten Module
+     * @param options
+     */
+    protected abstract initializeModule(options: IXeusWorkerKernel.IOptions): any;
+    /**
+     * Initialize the this.Module.FS as needed
+     * @param options
+     */
+    protected abstract initializeFileSystem(options: IXeusWorkerKernel.IOptions): Promise<any>;
+    /**
+     * Initialize the interpreter if needed
+     * @param options
+     */
+    protected abstract initializeInterpreter(options: IXeusWorkerKernel.IOptions): Promise<any>;
+    /**
+     * Add get_stdin function to globalThis that takes an input_request message, blocks
+     * until the corresponding input_reply is received and returns the input_reply message.
+     * If an error occurs return an object of the form { error: "Error explanation" }
+     * This function is called by xeus-lite's get_stdin.
+     */
+    protected abstract initializeStdin(baseUrl: string, browsingContextId: string): void;
+    /**
+     * Setup custom Emscripten FileSystem
+     */
+    abstract mount(driveName: string, mountpoint: string, baseUrl: string, browsingContextId: string): Promise<void>;
+    /**
+     * Implements dynamic installation of packages
+     */
+    protected abstract install(channels: string[], specs: string[], pipSpecs: string[]): Promise<void>;
+    /**
+     * Implements dynamic installation of packages
+     */
+    protected abstract listInstalledPackages(): Promise<void>;
+    /**
+     * Process magics prior to executing code
+     * @returns the runnable code without magics
+     */
+    protected processMagics(code: string): Promise<string>;
+    protected xkernel: any;
+    protected xserver: any;
+    protected setKernelReady: (value: void) => void;
+    private _ready;
+    private _logger;
+}
+export declare namespace XeusRemoteKernelBase {
+    interface IOptions {
+    }
+}

package/lib/worker.base.js

@@ -0,0 +1,185 @@
+// Copyright (c) Thorsten Beier
+// Copyright (c) JupyterLite Contributors
+// Distributed under the terms of the Modified BSD License.
+import { waitRunDependencies, parse } from '@emscripten-forge/mambajs';
+const STREAM = { log: 'stdout', warn: 'stdout', error: 'stderr' };
+export class XeusWorkerLoggerBase {
+    constructor(kernelId) {
+        this.executionCount = 0;
+        this._id = kernelId;
+        this._channel = new BroadcastChannel('/xeus-kernel-logs-broadcast');
+    }
+    log(...msg) {
+        postMessage({
+            _stream: {
+                name: STREAM['log'],
+                text: msg.join(' ') + '\n'
+            }
+        });
+        this._channel.postMessage({
+            kernelId: this._id,
+            payload: { type: 'text', level: 'info', data: msg.join(' ') }
+        });
+    }
+    warn(...msg) {
+        postMessage({
+            _stream: {
+                name: STREAM['warn'],
+                text: msg.join(' ') + '\n'
+            }
+        });
+        this._channel.postMessage({
+            kernelId: this._id,
+            payload: { type: 'text', level: 'warning', data: msg.join(' ') }
+        });
+    }
+    error(...msg) {
+        postMessage({
+            _stream: {
+                name: STREAM['error'],
+                evalue: msg.join(''),
+                traceback: [],
+                executionCount: this.executionCount,
+                text: msg.join('')
+            }
+        });
+        this._channel.postMessage({
+            kernelId: this._id,
+            payload: { type: 'text', level: 'critical', data: msg.join(' ') }
+        });
+    }
+}
+/**
+ * The base class for the worker kernel
+ *
+ * Meant to be extended in order to load kernels from other sources, implement custom magics etc.
+ */
+export class XeusRemoteKernelBase {
+    constructor(options = {}) {
+        this._ready = new Promise(resolve => {
+            this.setKernelReady = resolve;
+        });
+    }
+    async ready() {
+        return await this._ready;
+    }
+    async cd(path) {
+        if (!path || !this.Module.FS) {
+            return;
+        }
+        this.Module.FS.chdir(path);
+    }
+    async isDir(path) {
+        try {
+            const lookup = this.Module.FS.lookupPath(path);
+            return this.Module.FS.isDir(lookup.node.mode);
+        }
+        catch (e) {
+            return false;
+        }
+    }
+    async processMessage(event) {
+        const msg_type = event.msg.header.msg_type;
+        await globalThis.ready;
+        if (globalThis.toplevel_promise !== null &&
+            globalThis.toplevel_promise_py_proxy !== null) {
+            await globalThis.toplevel_promise;
+            globalThis.toplevel_promise_py_proxy.delete();
+            globalThis.toplevel_promise_py_proxy = null;
+            globalThis.toplevel_promise = null;
+        }
+        if (msg_type === 'input_reply') {
+            // Should never be called as input_reply messages are handled by get_stdin
+            // via SharedArrayBuffer or service worker.
+        }
+        else if (msg_type === 'execute_request') {
+            this._logger.executionCount += 1;
+            event.msg.content.code = await this.processMagics(event.msg.content.code);
+            this.xserver.notify_listener(event.msg);
+        }
+        else {
+            this.xserver.notify_listener(event.msg);
+        }
+    }
+    get Module() {
+        return globalThis.Module;
+    }
+    set Module(value) {
+        globalThis.Module = value;
+    }
+    get logger() {
+        return this._logger;
+    }
+    async initialize(options) {
+        const { baseUrl, browsingContextId, kernelSpec } = options;
+        this._logger = this.initializeLogger(options);
+        // when a toplevel cell uses an await, the cell is implicitly
+        // wrapped in a async function. Since the webloop - eventloop
+        // implementation does not support `eventloop.run_until_complete(f)`
+        // we need to convert the toplevel future in a javascript Promise
+        // this `toplevel` promise is then awaited before we
+        // execute the next cell. After the promise is awaited we need
+        // to do some cleanup and delete the python proxy
+        // (ie a js-wrapped python object) to avoid memory leaks
+        globalThis.toplevel_promise = null;
+        globalThis.toplevel_promise_py_proxy = null;
+        this.Module = await this.initializeModule(options);
+        this.Module = await createXeusModule(this.Module);
+        try {
+            await waitRunDependencies(this.Module);
+            await this.initializeFileSystem(options);
+            await this.initializeInterpreter(options);
+            this.initializeStdin(baseUrl, browsingContextId);
+            try {
+                this.xkernel = new this.Module.xkernel(kernelSpec.argv);
+            }
+            catch (e) {
+                this.xkernel = new this.Module.xkernel();
+            }
+            this.xserver = this.xkernel.get_server();
+            if (!this.xserver) {
+                this._logger.error('Failed to start kernel!');
+            }
+            this.xkernel.start();
+        }
+        catch (e) {
+            if (typeof e === 'number') {
+                const msg = this.Module.get_exception_message(e);
+                this._logger.error(msg);
+                throw new Error(msg);
+            }
+            else {
+                this._logger.error(e);
+                throw e;
+            }
+        }
+        this._logger.log('Kernel successfuly started!');
+        this.setKernelReady();
+    }
+    initializeLogger(options) {
+        return new XeusWorkerLoggerBase(options.kernelId);
+    }
+    /**
+     * Process magics prior to executing code
+     * @returns the runnable code without magics
+     */
+    async processMagics(code) {
+        const { commands, run } = parse(code);
+        for (const command of commands) {
+            switch (command.type) {
+                case 'install':
+                    if (command.data) {
+                        const { channels, specs, pipSpecs } = command.data;
+                        await this.install(channels, specs, pipSpecs);
+                    }
+                    break;
+                case 'list':
+                    await this.listInstalledPackages();
+                    break;
+                default:
+                    break;
+            }
+        }
+        return run;
+    }
+}

package/lib/worker.d.ts

@@ -1,17 +1,29 @@
 import { IXeusWorkerKernel } from './interfaces';
-export declare class XeusRemoteKernel {
-    constructor(options?: XeusRemoteKernel.IOptions);
-    ready(): Promise<void>;
-    cd(path: string): Promise<void>;
-    isDir(path: string): Promise<boolean>;
-    processMessage(event: any): Promise<void>;
-    initialize(options: IXeusWorkerKernel.IOptions): Promise<void>;
+import { XeusRemoteKernelBase } from './worker.base';
+/**
+ * A worker kernel that is backed by an empack environment
+ */
+export declare abstract class EmpackedXeusRemoteKernel extends XeusRemoteKernelBase {
+    protected initializeModule(options: IXeusWorkerKernel.IOptions): Promise<any>;
+    protected initializeFileSystem(options: IXeusWorkerKernel.IOptions): Promise<any>;
     /**
-     * Register the callback function to send messages from the worker back to the main thread.
-     * @param callback the callback to register
+     * Initialize the interpreter if needed
+     * @param options
      */
-    registerCallback(callback: (msg: any) => void): void;
-    protected _sendWorkerMessage: (msg: any) => void;
+    protected initializeInterpreter(options: IXeusWorkerKernel.IOptions): Promise<void>;
+    protected install(channels: string[], specs: string[], pipSpecs: string[]): Promise<void>;
+    /**
+     * Implements dynamic installation of packages
+     */
+    protected listInstalledPackages(): Promise<void>;
+    private _reloadPackagesInFS;
+    private _pythonVersion;
+    private _prefix;
+    private _pkgRootUrl;
+    private _sharedLibs;
+    private _installedPackages;
+    private _paths;
+    private _untarjs;
 }
 export declare namespace XeusRemoteKernel {
     interface IOptions {