jupyterpack 0.2.0 → 0.3.0

package/lib/pythonWidget/pythonWidgetModel.js

@@ -1,6 +1,6 @@
-import { PromiseDelegate } from '@lumino/coreutils';
-import { KernelExecutor } from './kernelExecutor';
 import { PathExt } from '@jupyterlab/coreutils';
+import { PromiseDelegate } from '@lumino/coreutils';
+import { PYTHON_SERVER } from '../pythonServer';
 export class PythonWidgetModel {
     constructor(options) {
         this._isDisposed = false;
@@ -9,6 +9,7 @@ export class PythonWidgetModel {
         this._manager = options.manager;
         this._connectionManager = options.connectionManager;
         this._contentsManager = options.contentsManager;
+        this._jpackModel = options.jpackModel;
     }
     get isDisposed() {
         return this._isDisposed;
@@ -17,13 +18,17 @@ export class PythonWidgetModel {
         return this._connectionManager;
     }
     async initialize() {
-        var _a;
+        var _a, _b;
         if (this._kernelStarted) {
-            return null;
+            return {
+                success: false,
+                error: 'Server is called twice'
+            };
         }
         const filePath = this._context.localPath;
-        const spkContent = this._context.model.toJSON();
+        const spkContent = this._jpackModel;
         const entryPath = PathExt.join(PathExt.dirname(filePath), spkContent.entry);
+        const rootUrl = (_a = spkContent.rootUrl) !== null && _a !== void 0 ? _a : '/';
         const entryContent = await this._contentsManager.get(entryPath, {
             content: true,
             format: 'text'
@@ -33,7 +38,10 @@ export class PythonWidgetModel {
         await this._manager.kernelspecs.ready;
         const specs = this._manager.kernelspecs.specs;
         if (!specs) {
-            return null;
+            return {
+                success: false,
+                error: 'Missing kernel spec'
+            };
         }
         const { kernelspecs } = specs;
         let kernelName = Object.keys(kernelspecs)[0];
@@ -48,12 +56,21 @@ export class PythonWidgetModel {
             },
             type: 'notebook'
         });
-        const executor = new KernelExecutor({
+        const framework = spkContent.framework;
+        const ServerClass = PYTHON_SERVER.get(framework);
+        if (!ServerClass) {
+            return {
+                success: false,
+                error: `Framework "${framework}" is not supported. Please check your .spk file.`
+            };
+        }
+        const executor = (this._executor = new ServerClass({
             sessionConnection: this._sessionConnection
-        });
+        }));
         const data = await this._connectionManager.registerConnection(executor);
         await executor.init({
             initCode: entryContent.content,
+            entryPath: spkContent.entry,
             ...data
         });
         const finish = new PromiseDelegate();
@@ -64,12 +81,17 @@ export class PythonWidgetModel {
                 finish.resolve();
             }
         };
-        (_a = this._sessionConnection.kernel) === null || _a === void 0 ? void 0 : _a.statusChanged.connect(cb);
+        (_b = this._sessionConnection.kernel) === null || _b === void 0 ? void 0 : _b.statusChanged.connect(cb);
         await finish.promise;
         this._kernelStarted = true;
-        return data;
+        return { ...data, rootUrl, framework, success: true };
     }
     dispose() {
+        var _a;
+        if (this._isDisposed) {
+            return;
+        }
+        void ((_a = this._executor) === null || _a === void 0 ? void 0 : _a.disposePythonServer());
         this._isDisposed = true;
     }
 }

package/lib/swConnection/index.js

@@ -3,7 +3,7 @@ import { UUID } from '@lumino/coreutils';
 import { expose } from 'comlink';
 import { IConnectionManagerToken } from '../token';
 import { MessageAction } from '../type';
-import { ConnectionManager } from './connection_manager';
+import { ConnectionManager } from './mainConnectionManager';
 const fullLabextensionsUrl = PageConfig.getOption('fullLabextensionsUrl');
 const SCOPE = `${fullLabextensionsUrl}/jupyterpack/static`;
 async function initServiceWorker() {
@@ -50,12 +50,12 @@ export const swPlugin = {
     autoStart: true,
     provides: IConnectionManagerToken,
     activate: async (app) => {
-        console.log('Activating jupyterpack service worker');
         const serviceWorker = await initServiceWorker();
         if (!serviceWorker) {
             throw new Error('Failed to register the Service Worker, please make sure to use a browser that supports this feature.');
         }
         const instanceId = UUID.uuid4();
+        console.log('Activating jupyterpack service worker with instance id', instanceId);
         const { port1: mainToServiceWorker, port2: serviceWorkerToMain } = new MessageChannel();
         const connectionManager = new ConnectionManager(instanceId);
         expose(connectionManager, mainToServiceWorker);

package/lib/{pythonWidget/connectionManager.d.ts → swConnection/mainConnectionManager.d.ts}

@@ -1,4 +1,12 @@
 import { IConnectionManager, IDict, IKernelExecutor } from '../type';
+/**
+ * Manages connections between clients and kernel executors.
+ * This class handles the registration of kernel executors and the generation of responses
+ * for client requests. It maintains a mapping of kernel client IDs to their respective executors.
+ * The HTTP requests intercepted by the service worker are forwarded to the appropriate kernel executor.
+ * The websocket messages forwarded from the broadcast channel are also forwarded to the appropriate kernel executor.
+ * It's running on the main thread
+ */
 export declare class ConnectionManager implements IConnectionManager {
     instanceId: string;
     constructor(instanceId: string);
@@ -14,5 +22,7 @@ export declare class ConnectionManager implements IConnectionManager {
         requestBody?: ArrayBuffer;
         params?: string;
     }): Promise<IDict | null>;
+    private _initWsChannel;
     private _kernelExecutors;
+    private _wsBroadcastChannel;
 }

package/lib/swConnection/mainConnectionManager.js

@@ -0,0 +1,93 @@
+import { arrayBufferToBase64 } from '../tools';
+import { UUID } from '@lumino/coreutils';
+/**
+ * Manages connections between clients and kernel executors.
+ * This class handles the registration of kernel executors and the generation of responses
+ * for client requests. It maintains a mapping of kernel client IDs to their respective executors.
+ * The HTTP requests intercepted by the service worker are forwarded to the appropriate kernel executor.
+ * The websocket messages forwarded from the broadcast channel are also forwarded to the appropriate kernel executor.
+ * It's running on the main thread
+ */
+export class ConnectionManager {
+    constructor(instanceId) {
+        this.instanceId = instanceId;
+        this._kernelExecutors = new Map();
+        this._wsBroadcastChannel = new BroadcastChannel(`/jupyterpack/ws/${instanceId}`);
+        this._initWsChannel();
+    }
+    async registerConnection(kernelExecutor) {
+        const uuid = UUID.uuid4();
+        this._kernelExecutors.set(uuid, kernelExecutor);
+        return { instanceId: this.instanceId, kernelClientId: uuid };
+    }
+    async generateResponse(options) {
+        const { urlPath, kernelClientId, method, params, requestBody, headers } = options;
+        const executor = this._kernelExecutors.get(kernelClientId);
+        if (!executor) {
+            return null;
+        }
+        const response = await executor.getResponse({
+            urlPath,
+            method,
+            params,
+            headers,
+            requestBody
+        });
+        return response;
+    }
+    _initWsChannel() {
+        this._wsBroadcastChannel.onmessage = event => {
+            const rawData = event.data;
+            let data;
+            if (typeof rawData === 'string') {
+                data = JSON.parse(rawData);
+            }
+            else {
+                data = rawData;
+            }
+            const { action, dest, wsUrl, payload } = data;
+            const executor = this._kernelExecutors.get(dest);
+            if (!executor) {
+                console.error('Missing kernel handle for message', data, dest, this._kernelExecutors);
+                return;
+            }
+            switch (action) {
+                case 'open': {
+                    executor.openWebsocket({
+                        instanceId: this.instanceId,
+                        kernelId: dest,
+                        wsUrl,
+                        protocol: payload.protocol
+                    });
+                    break;
+                }
+                case 'send': {
+                    let serializedData;
+                    let isBinary;
+                    if (payload instanceof ArrayBuffer || ArrayBuffer.isView(payload)) {
+                        // Convert data to base64 string
+                        serializedData = arrayBufferToBase64(payload);
+                        isBinary = true;
+                    }
+                    else if (typeof payload === 'string') {
+                        serializedData = payload;
+                        isBinary = false;
+                    }
+                    else {
+                        console.error('Unknown message type', payload);
+                        return;
+                    }
+                    executor.sendWebsocketMessage({
+                        instanceId: this.instanceId,
+                        kernelId: dest,
+                        wsUrl,
+                        message: JSON.stringify({ isBinary, data: serializedData })
+                    });
+                    break;
+                }
+                default:
+                    break;
+            }
+        };
+    }
+}

package/lib/swConnection/sw.js

@@ -1,6 +1,6 @@
 // import { expose } from 'comlink';
 import { MessageAction } from '../type';
-import { CommManager } from './comm_manager';
+import { CommManager } from './swCommManager';
 const COMM_MANAGER = new CommManager();
 /**
  * Install event listeners
@@ -32,7 +32,10 @@ async function onFetch(event) {
     if (url.endsWith('__jupyterpack__/ping.html')) {
         return;
     }
-    event.respondWith(COMM_MANAGER.generateResponse(event.request));
+    if (url.includes('extensions/jupyterpack/static')) {
+        return event.respondWith(COMM_MANAGER.generateResponse(event.request));
+    }
+    return;
 }
 function onMessage(msg) {
     const { type, data } = msg.data;

package/lib/swConnection/swCommManager.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Manages communication between different components using Comlink's MessagePort-based communication.
+ * This class handles registration of communication channels and processing of requests.
+ * It's running on the service worker thread
+ */
+export declare class CommManager {
+    constructor();
+    registerComm(instanceId: string, port: MessagePort): void;
+    generateResponse(request: Request): Promise<Response>;
+    private _commIds;
+}

package/lib/swConnection/{comm_manager.js → swCommManager.js}

@@ -1,4 +1,9 @@
 import { wrap, transfer } from 'comlink';
+/**
+ * Manages communication between different components using Comlink's MessagePort-based communication.
+ * This class handles registration of communication channels and processing of requests.
+ * It's running on the service worker thread
+ */
 export class CommManager {
     constructor() {
         this._commIds = new Map();
@@ -19,9 +24,12 @@ export class CommManager {
         }
         const params = url.searchParams.toString();
         const pathAfterExtensionName = urlPath.split('/jupyterpack/static')[1];
-        const pathList = pathAfterExtensionName.split('/').filter(Boolean);
-        const instanceId = pathList[0];
-        const kernelClientId = pathList[2];
+        const pathList = pathAfterExtensionName === null || pathAfterExtensionName === void 0 ? void 0 : pathAfterExtensionName.split('/').filter(Boolean);
+        const instanceId = pathList === null || pathList === void 0 ? void 0 : pathList[0];
+        const kernelClientId = pathList === null || pathList === void 0 ? void 0 : pathList[2];
+        if (!instanceId || !kernelClientId) {
+            return await fetch(url, { method });
+        }
         const comm = this._commIds.get(instanceId);
         if (!comm) {
             throw new Error('Missing comm');

package/lib/tools.d.ts

@@ -1,2 +1,6 @@
 export declare function removePrefix(path: string, prefix: string): string;
 export declare function arrayBufferToBase64(buffer: ArrayBuffer): string;
+export declare function base64ToArrayBuffer(base64: string): Uint8Array;
+export declare function base64ToString(base64: string): string;
+export declare function stringOrNone(content?: string): string;
+export declare function isBinaryContentType(contentType?: string): boolean;

package/lib/tools.js

@@ -15,3 +15,61 @@ export function arrayBufferToBase64(buffer) {
     }
     return btoa(binary);
 }
+export function base64ToArrayBuffer(base64) {
+    const binaryString = atob(base64);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes;
+}
+export function base64ToString(base64) {
+    const bytes = base64ToArrayBuffer(base64);
+    return new TextDecoder('utf-8').decode(bytes);
+}
+export function stringOrNone(content) {
+    return content ? `"${content}"` : 'None';
+}
+export function isBinaryContentType(contentType) {
+    if (!contentType) {
+        // no Content-Type → assume binary for safety
+        return true;
+    }
+    contentType = contentType.toLowerCase().trim();
+    const textTypes = [
+        'text/',
+        'application/json',
+        'application/javascript',
+        'application/xml',
+        'application/xhtml+xml',
+        'application/x-www-form-urlencoded',
+        'application/sql',
+        'application/graphql',
+        'application/yaml'
+    ];
+    const binaryIndicators = [
+        'image/',
+        'audio/',
+        'video/',
+        'font/',
+        'application/octet-stream',
+        'application/pdf',
+        'application/zip',
+        'application/x-protobuf',
+        'application/vnd'
+    ];
+    // Starts with text/ or one of the textual types
+    if (textTypes.some(t => contentType.startsWith(t))) {
+        return false;
+    }
+    // Starts with binary-indicating prefix
+    if (binaryIndicators.some(t => contentType.startsWith(t))) {
+        return true;
+    }
+    // If charset is specified → text
+    if (contentType.includes('charset=')) {
+        return false;
+    }
+    // Default: assume binary
+    return true;
+}

package/lib/type.d.ts

@@ -3,15 +3,24 @@ import { IDisposable } from '@lumino/disposable';
 export interface IDict<T = any> {
     [key: string]: T;
 }
+export interface IBroadcastMessage {
+    action: 'message' | 'open' | 'close' | 'error' | 'send' | 'connected' | 'backend_message';
+    dest: string;
+    wsUrl: string;
+    payload?: any;
+}
 export declare enum JupyterPackFramework {
     REACT = "react",
-    DASH = "dash"
+    DASH = "dash",
+    STREAMLIT = "streamlit",
+    TORNADO = "tornado"
 }
 export interface IJupyterPackFileFormat {
     entry: string;
     framework: JupyterPackFramework;
     name?: string;
     metadata?: IDict;
+    rootUrl?: string;
 }
 export declare enum MessageAction {
     INIT = "INIT"
@@ -25,7 +34,33 @@ export interface IKernelExecutorParams {
 }
 export interface IKernelExecutor extends IDisposable {
     getResponse(options: IKernelExecutorParams): Promise<IDict>;
-    executeCode(code: KernelMessage.IExecuteRequestMsg['content']): Promise<string>;
+    openWebsocket(options: {
+        instanceId: string;
+        kernelId: string;
+        wsUrl: string;
+        protocol?: string;
+    }): Promise<void>;
+    sendWebsocketMessage(options: {
+        instanceId: string;
+        kernelId: string;
+        wsUrl: string;
+        message: string;
+    }): Promise<void>;
+    executeCode(code: KernelMessage.IExecuteRequestMsg['content'], waitForResult?: boolean): Promise<string | null>;
+    init(options: {
+        entryPath?: string;
+        initCode?: string;
+        instanceId: string;
+        kernelClientId: string;
+    }): Promise<void>;
+    disposePythonServer(): Promise<void>;
+    getResponseFunctionFactory(options: {
+        urlPath: string;
+        method: string;
+        headers: IDict;
+        params?: string;
+        content?: string;
+    }): string;
 }
 export interface IConnectionManager {
     registerConnection(kernelExecutor: IKernelExecutor): Promise<{

package/lib/type.js

@@ -2,6 +2,8 @@ export var JupyterPackFramework;
 (function (JupyterPackFramework) {
     JupyterPackFramework["REACT"] = "react";
     JupyterPackFramework["DASH"] = "dash";
+    JupyterPackFramework["STREAMLIT"] = "streamlit";
+    JupyterPackFramework["TORNADO"] = "tornado";
 })(JupyterPackFramework || (JupyterPackFramework = {}));
 export var MessageAction;
 (function (MessageAction) {

package/lib/websocket/websocket.js

@@ -0,0 +1,152 @@
+"use strict";
+(() => {
+    const urlPath = new URL(window.location.href).pathname;
+    const pathAfterExtensionName = urlPath.split('/jupyterpack/static')[1];
+    const pathList = pathAfterExtensionName === null || pathAfterExtensionName === void 0 ? void 0 : pathAfterExtensionName.split('/').filter(Boolean);
+    const instanceId = pathList === null || pathList === void 0 ? void 0 : pathList[0];
+    const kernelClientId = pathList === null || pathList === void 0 ? void 0 : pathList[2];
+    if (!instanceId || !kernelClientId) {
+        throw new Error('Missing instance Id or kernelClient Id');
+    }
+    const sendTypedMessage = (msg) => {
+        bcWsChannel.postMessage({ ...msg, dest: kernelClientId });
+    };
+    function base64ToBinary(base64, dataType) {
+        const binary = atob(base64); // decode base64
+        const len = binary.length;
+        const bytes = new Uint8Array(len);
+        for (let i = 0; i < len; i++) {
+            bytes[i] = binary.charCodeAt(i);
+        }
+        if (dataType === 'arraybuffer') {
+            return bytes.buffer;
+        }
+        else if (dataType === 'blob') {
+            return new Blob([bytes], { type: 'application/octet-stream' });
+        }
+        else {
+            throw new Error("Unsupported type: use 'arraybuffer' or 'blob'");
+        }
+    }
+    const decodeServerMessage = (payload, binaryType) => {
+        const { data, isBinary } = payload;
+        if (isBinary) {
+            // Decode base64 string to array buffer or blob
+            return base64ToBinary(data, binaryType);
+        }
+        return data;
+    };
+    const bcWsChannel = new BroadcastChannel(`/jupyterpack/ws/${instanceId}`);
+    class BroadcastChannelWebSocket {
+        constructor(url, protocols) {
+            this.onclose = () => {
+                // no-op
+            };
+            this.onerror = () => {
+                // no-op
+            };
+            this.onmessage = () => {
+                // no-op
+            };
+            this.onopen = () => {
+                // no-op
+            };
+            this.CONNECTING = 0;
+            this.OPEN = 1;
+            this.CLOSING = 2;
+            this.CLOSED = 3;
+            this._eventHandlers = { message: [], open: [], close: [], error: [] };
+            this._bcMessageHandler = async (event) => {
+                const rawData = event.data;
+                let data;
+                if (typeof rawData === 'string') {
+                    data = JSON.parse(rawData);
+                }
+                else {
+                    data = rawData;
+                }
+                const { action, dest, wsUrl, payload } = data;
+                if (dest !== kernelClientId || wsUrl !== this.url) {
+                    return;
+                }
+                switch (action) {
+                    case 'connected': {
+                        this.readyState = this.OPEN;
+                        if (this.onopen) {
+                            this.onopen(event);
+                        }
+                        this._eventHandlers.open.forEach(handler => handler({ data: payload }));
+                        break;
+                    }
+                    case 'backend_message': {
+                        const decoded = decodeServerMessage(payload, this.binaryType);
+                        if (this.onmessage) {
+                            this.onmessage({ data: decoded });
+                        }
+                        this._eventHandlers.message.forEach(handler => handler({ data: decoded }));
+                        break;
+                    }
+                    default:
+                        break;
+                }
+            };
+            this._open = () => {
+                sendTypedMessage({
+                    action: 'open',
+                    payload: {
+                        protocol: this.protocol
+                    },
+                    wsUrl: this.url
+                });
+                bcWsChannel.addEventListener('message', this._bcMessageHandler);
+            };
+            const urlObj = new URL(url);
+            this.url = urlObj.pathname + urlObj.search + urlObj.hash;
+            if (protocols) {
+                this.protocol = Array.isArray(protocols)
+                    ? protocols.join(',')
+                    : protocols;
+            }
+            else {
+                this.protocol = '';
+            }
+            this.binaryType = 'blob';
+            this.bufferedAmount = 0;
+            this.extensions = '';
+            this.readyState = this.CONNECTING;
+            this._open();
+        }
+        close(code, reason) {
+            if (this.readyState === this.CLOSED) {
+                return;
+            }
+            if (this.onclose) {
+                this.onclose();
+            }
+            while (this._eventHandlers['close'].length) {
+                const cb = this._eventHandlers['close'].pop();
+                cb();
+            }
+            this._eventHandlers['close'] = [];
+            bcWsChannel.removeEventListener('message', this._bcMessageHandler);
+            this.readyState = this.CLOSED;
+        }
+        send(data) {
+            sendTypedMessage({
+                action: 'send',
+                payload: data,
+                wsUrl: this.url
+            });
+        }
+        addEventListener(type, listener, options) {
+            this._eventHandlers[type].push(listener);
+        }
+        removeEventListener(type, listener, options) {
+            this._eventHandlers[type] = this._eventHandlers[type].filter(handler => handler !== listener);
+        }
+        dispatchEvent(event) {
+            return true;
+        }
+    }
+    window.WebSocket = BroadcastChannelWebSocket;
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "jupyterpack",
-    "version": "0.2.0",
+    "version": "0.3.0",
     "description": "A JupyterLab extension for serving web app.",
     "keywords": [
         "jupyter",
@@ -31,10 +31,11 @@
     "scripts": {
         "build:demo": "cd demo && rm -rf .jupyterlite.doit.db _output && jupyter lite build .",
         "update:demo": "node script/update-demo.mjs",
+        "combine:python": "node script/combine-python.mjs",
         "build:worker": "webpack --config sw.webpack.config.js --mode development",
         "build:worker:prod": "webpack --config sw.webpack.config.js --mode production",
-        "build": "jlpm build:lib && jlpm build:labextension:dev && jlpm build:worker",
-        "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension && jlpm build:worker:prod",
+        "build": "jlpm combine:python && jlpm build:lib && jlpm build:labextension:dev && jlpm build:worker",
+        "build:prod": "jlpm clean && jlpm combine:python && jlpm build:lib:prod && jlpm build:labextension && jlpm build:worker:prod",
         "build:labextension": "jupyter labextension build .",
         "build:labextension:dev": "jupyter labextension build --development True .",
         "build:lib": "tsc --sourceMap",
@@ -61,7 +62,8 @@
     "dependencies": {
         "@codesandbox/sandpack-client": "^2.19.8",
         "@jupyterlab/application": "^4.0.0",
-        "comlink": "^4.4.2"
+        "comlink": "^4.4.2",
+        "strip-ansi": "^7.1.2"
     },
     "devDependencies": {
         "@jupyterlab/builder": "^4.0.0",
@@ -102,7 +104,8 @@
     },
     "jupyterlab": {
         "extension": true,
-        "outputDir": "jupyterpack/labextension"
+        "outputDir": "jupyterpack/labextension",
+        "webpackConfig": "./extension.webpack.config.js"
     },
     "eslintIgnore": [
         "node_modules",

package/src/document/widgetFactory.ts

@@ -49,8 +49,11 @@ export class JupyterPackWidgetFactory extends ABCWidgetFactory<JupyterPackDocWid
           content.addWidget(jpContent);
           break;
         }
-        case JupyterPackFramework.DASH: {
+        case JupyterPackFramework.DASH:
+        case JupyterPackFramework.STREAMLIT:
+        case JupyterPackFramework.TORNADO: {
           const model = new PythonWidgetModel({
+            jpackModel,
             context,
             manager: this.options.manager,
             contentsManager: this._contentsManager,

package/src/global.d.ts

@@ -0,0 +1,4 @@
+declare module '*?raw' {
+  const content: string;
+  export default content;
+}