jupyterpack 0.2.1 → 0.4.0

package/src/pythonWidget/comm.ts

@@ -0,0 +1,65 @@
+import { Kernel, KernelMessage } from '@jupyterlab/services';
+
+const COMM_NAME = 'jupyterpack:broadcast:comm';
+
+export class CommBroadcastManager {
+  constructor() {
+    this._kernels = new Map();
+    this._comms = new Map();
+  }
+
+  registerKernel(kernel: Kernel.IKernelConnection) {
+    this._kernels.set(kernel.id, kernel);
+    kernel.registerCommTarget(COMM_NAME, (comm, msg) =>
+      this._handle_comm_open(comm, msg, kernel.id)
+    );
+  }
+  unregisterKernel(kernelId?: string) {
+    if (kernelId) {
+      this._kernels.delete(kernelId);
+      const comms = this._comms.get(kernelId) ?? [];
+      comms.forEach(comm => comm.dispose());
+      this._comms.delete(kernelId);
+    }
+  }
+
+  private _handle_comm_open = async (
+    comm: Kernel.IComm,
+    msg: KernelMessage.ICommOpenMsg,
+    kernelId: string
+  ): Promise<void> => {
+    if (this._comms.has(kernelId)) {
+      this._comms.get(kernelId)?.push(comm);
+    } else {
+      this._comms.set(kernelId, [comm]);
+    }
+    const channelName = msg.metadata.channel_name as string | undefined;
+    if (!channelName) {
+      return;
+    }
+    if (!this._broadcastChannels.has(channelName)) {
+      this._broadcastChannels.set(
+        channelName,
+        new BroadcastChannel(channelName)
+      );
+    }
+    const broadcastChannel = this._broadcastChannels.get(channelName)!;
+    comm.onMsg = commMsg => {
+      const { data } = commMsg.content;
+      broadcastChannel.postMessage(data);
+    };
+  };
+
+  dispose() {
+    this._kernels.clear();
+    this._comms.clear();
+    this._broadcastChannels.forEach(it => {
+      it.close();
+    });
+    this._broadcastChannels.clear();
+  }
+
+  private _kernels: Map<string, Kernel.IKernelConnection> = new Map();
+  private _comms: Map<string, Kernel.IComm[]> = new Map();
+  private _broadcastChannels: Map<string, BroadcastChannel> = new Map();
+}

package/src/pythonWidget/pythonWidget.ts

@@ -1,29 +1,64 @@
 import { PythonWidgetModel } from './pythonWidgetModel';
-import { PageConfig } from '@jupyterlab/coreutils';
+import { PageConfig, URLExt } from '@jupyterlab/coreutils';
 import { IFramePanel } from '../document/iframePanel';
+import { PromiseDelegate } from '@lumino/coreutils';
 
 export class PythonWidget extends IFramePanel {
   constructor(options: PythonWidget.IOptions) {
     super();
     this._model = options.model;
     this._model.initialize().then(connectionData => {
-      if (!connectionData) {
+      if (!connectionData.success) {
+        this.toggleSpinner(false);
+        this._iframe.contentDocument!.body.innerText = `Failed to start server: ${connectionData.error}`;
         return;
       }
+      this._isReady.resolve();
       const iframe = this._iframe;
       const fullLabextensionsUrl = PageConfig.getOption('fullLabextensionsUrl');
-      iframe.src = `${fullLabextensionsUrl}/jupyterpack/static/${connectionData.instanceId}/dash/${connectionData.kernelClientId}/`;
+
+      const iframeUrl = URLExt.join(
+        fullLabextensionsUrl,
+        'jupyterpack/static',
+        connectionData.instanceId,
+        connectionData.framework,
+        connectionData.kernelClientId,
+        connectionData.rootUrl
+      );
+      iframe.src = iframeUrl;
+
       iframe.addEventListener('load', () => {
         this.toggleSpinner(false);
       });
+      this._model.serverReloaded.connect(() => {
+        this._iframe?.contentWindow?.location?.reload();
+      });
     });
   }
 
+  get autoreload() {
+    return this._model.autoreload;
+  }
+  set autoreload(value: boolean) {
+    this._model.autoreload = value;
+  }
+  get isReady(): Promise<void> {
+    return this._isReady.promise;
+  }
   get model(): PythonWidgetModel {
     return this._model;
   }
 
+  async reload(): Promise<void> {
+    await this._model.reload();
+  }
+
+  dispose(): void {
+    this._model.dispose();
+  }
+
   private _model: PythonWidgetModel;
+  private _isReady = new PromiseDelegate<void>();
 }
 
 export namespace PythonWidget {

package/src/pythonWidget/pythonWidgetModel.ts

@@ -1,53 +1,98 @@
+import { PathExt } from '@jupyterlab/coreutils';
 import { DocumentRegistry } from '@jupyterlab/docregistry';
-import { IDisposable } from '@lumino/disposable';
 import {
-  ServiceManager,
-  Session,
+  Contents,
   Kernel,
-  Contents
+  ServiceManager,
+  Session
 } from '@jupyterlab/services';
 import { PromiseDelegate } from '@lumino/coreutils';
-import { IConnectionManager, IJupyterPackFileFormat } from '../type';
-import { KernelExecutor } from './kernelExecutor';
-import { PathExt } from '@jupyterlab/coreutils';
 
-export class PythonWidgetModel implements IDisposable {
+import { PYTHON_SERVER } from '../pythonServer';
+import { Signal } from '@lumino/signaling';
+import {
+  IConnectionManager,
+  IJupyterPackFileFormat,
+  IKernelExecutor,
+  IPythonWidgetModel,
+  JupyterPackFramework
+} from '../type';
+import { CommBroadcastManager } from './comm';
+import { IS_LITE } from '../tools';
+
+export class PythonWidgetModel implements IPythonWidgetModel {
   constructor(options: PythonWidgetModel.IOptions) {
     this._context = options.context;
     this._manager = options.manager;
     this._connectionManager = options.connectionManager;
     this._contentsManager = options.contentsManager;
+    this._jpackModel = options.jpackModel;
+    this._localPath = PathExt.dirname(this._context.localPath);
+    this._autoreload = Boolean(this._jpackModel?.metadata?.autoreload);
+
+    this._contentsManager.fileChanged.connect(this._onFileChanged, this);
+  }
+
+  get autoreload() {
+    return this._autoreload;
   }
 
+  set autoreload(val: boolean) {
+    this._autoreload = val;
+  }
   get isDisposed(): boolean {
     return this._isDisposed;
   }
   get connectionManager(): IConnectionManager {
     return this._connectionManager;
   }
-  async initialize(): Promise<{
-    instanceId: string;
-    kernelClientId: string;
-  } | null> {
-    if (this._kernelStarted) {
-      return null;
+  get serverReloaded() {
+    return this._serverReloaded;
+  }
+  get kernelStatusChanged() {
+    return this._kernelStatusChanged;
+  }
+
+  async reload() {
+    if (!this._kernelStarted) {
+      return;
     }
-    const filePath = this._context.localPath;
-    const spkContent =
-      this._context.model.toJSON() as any as IJupyterPackFileFormat;
+    const { spkContent, entryContent } = await this._loadData();
+    await this._executor?.reloadPythonServer({
+      entryPath: spkContent.entry,
+      initCode: entryContent.content
+    });
+    this._serverReloaded.emit();
+  }
 
-    const entryPath = PathExt.join(PathExt.dirname(filePath), spkContent.entry);
+  async initialize(): Promise<
+    | {
+        success: true;
+        instanceId: string;
+        kernelClientId: string;
+        rootUrl: string;
+        framework: JupyterPackFramework;
+      }
+    | { success: false; error: string }
+  > {
+    if (this._kernelStarted) {
+      return {
+        success: false,
+        error: 'Server is called twice'
+      };
+    }
 
-    const entryContent = await this._contentsManager.get(entryPath, {
-      content: true,
-      format: 'text'
-    });
+    const { filePath, spkContent, rootUrl, entryContent } =
+      await this._loadData();
     const sessionManager = this._manager.sessions;
     await sessionManager.ready;
     await this._manager.kernelspecs.ready;
     const specs = this._manager.kernelspecs.specs;
     if (!specs) {
-      return null;
+      return {
+        success: false,
+        error: 'Missing kernel spec'
+      };
     }
     const { kernelspecs } = specs;
     let kernelName = Object.keys(kernelspecs)[0];
@@ -55,20 +100,44 @@ export class PythonWidgetModel implements IDisposable {
       kernelName = specs.default;
     }
 
-    this._sessionConnection = await sessionManager.startNew({
-      name: filePath,
-      path: filePath,
-      kernel: {
-        name: kernelName
+    this._sessionConnection = await sessionManager.startNew(
+      {
+        name: filePath,
+        path: filePath,
+        kernel: {
+          name: kernelName
+        },
+        type: 'notebook'
       },
-      type: 'notebook'
-    });
-    const executor = new KernelExecutor({
+      {
+        kernelConnectionOptions: { handleComms: true }
+      }
+    );
+    const kernel = this._sessionConnection.kernel;
+    if (kernel) {
+      this._kernelStatusChanged.emit('started');
+      this._commBroadcastManager.registerKernel(kernel);
+      kernel.disposed.connect(() => {
+        this._kernelStatusChanged.emit('stopped');
+        this._commBroadcastManager.unregisterKernel(kernel.id);
+      });
+    }
+
+    const framework = spkContent.framework;
+    const ServerClass = PYTHON_SERVER.get(framework);
+    if (!ServerClass) {
+      return {
+        success: false,
+        error: `Framework "${framework}" is not supported. Please check your .spk file.`
+      };
+    }
+    const executor = (this._executor = new ServerClass({
       sessionConnection: this._sessionConnection
-    });
+    }));
     const data = await this._connectionManager.registerConnection(executor);
     await executor.init({
       initCode: entryContent.content,
+      entryPath: spkContent.entry,
       ...data
     });
     const finish = new PromiseDelegate<void>();
@@ -82,12 +151,50 @@ export class PythonWidgetModel implements IDisposable {
 
     await finish.promise;
     this._kernelStarted = true;
-    return data;
+    return { ...data, rootUrl, framework, success: true };
   }
-  dispose(): void {
+  async dispose(): Promise<void> {
+    if (this._isDisposed) {
+      return;
+    }
+    if (!IS_LITE) {
+      this._sessionConnection?.kernel?.shutdown();
+    }
+    void this._executor?.disposePythonServer();
+    this._contentsManager.fileChanged.disconnect(this._onFileChanged);
+    this._commBroadcastManager.dispose();
     this._isDisposed = true;
   }
 
+  private async _loadData() {
+    const filePath = this._context.localPath;
+    const spkContent = this._jpackModel;
+
+    const entryPath = PathExt.join(PathExt.dirname(filePath), spkContent.entry);
+    const rootUrl = spkContent.rootUrl ?? '/';
+    const entryContent = await this._contentsManager.get(entryPath, {
+      content: true,
+      format: 'text'
+    });
+    return { filePath, spkContent, rootUrl, entryContent };
+  }
+
+  private async _onFileChanged(
+    sender: Contents.IManager,
+    args: Contents.IChangedArgs
+  ) {
+    if (this._autoreload && args.type === 'save') {
+      if (
+        args.newValue?.path &&
+        args.newValue.path.startsWith(this._localPath)
+      ) {
+        await this.reload();
+      }
+    }
+  }
+
+  private _commBroadcastManager = new CommBroadcastManager();
+
   private _isDisposed = false;
   private _kernelStarted = false;
   private _sessionConnection: Session.ISessionConnection | undefined;
@@ -95,10 +202,24 @@ export class PythonWidgetModel implements IDisposable {
   private _context: DocumentRegistry.IContext<DocumentRegistry.IModel>;
   private _connectionManager: IConnectionManager;
   private _contentsManager: Contents.IManager;
+  private _jpackModel: IJupyterPackFileFormat;
+  private _executor?: IKernelExecutor;
+  private _localPath: string;
+
+  private _serverReloaded: Signal<IPythonWidgetModel, void> = new Signal<
+    IPythonWidgetModel,
+    void
+  >(this);
+  private _kernelStatusChanged: Signal<
+    IPythonWidgetModel,
+    'started' | 'stopped'
+  > = new Signal(this);
+  private _autoreload: boolean;
 }
 
 export namespace PythonWidgetModel {
   export interface IOptions {
+    jpackModel: IJupyterPackFileFormat;
     context: DocumentRegistry.IContext<DocumentRegistry.IModel>;
     manager: ServiceManager.IManager;
     connectionManager: IConnectionManager;

package/src/sandpackWidget/sandpackFilesModel.ts

@@ -3,16 +3,17 @@ import { ISignal, Signal } from '@lumino/signaling';
 
 import { IDict } from '../type';
 import { removePrefix } from '../tools';
+import { IDisposable } from '@lumino/disposable';
 
-export class SandpackFilesModel {
+export class SandpackFilesModel implements IDisposable {
   constructor(options: { contentsManager: Contents.IManager; path: string }) {
     this._contentManager = options.contentsManager;
     this._path = options.path;
     this._contentManager.fileChanged.connect(this._onFileChanged, this);
   }
 
-  async getAllFiles(): Promise<IDict<{ code: string }>> {
-    if (!this._allFiles) {
+  async getAllFiles(force = false): Promise<IDict<{ code: string }>> {
+    if (!this._allFiles || force) {
       const files = await this._contentManager.get(this._path, {
         content: true
       });
@@ -21,6 +22,9 @@ export class SandpackFilesModel {
 
     return this._allFiles;
   }
+  get isDisposed(): boolean {
+    return this._isDisposed;
+  }
 
   get fileChanged(): ISignal<
     SandpackFilesModel,
@@ -29,6 +33,14 @@ export class SandpackFilesModel {
     return this._fileChanged;
   }
 
+  dispose(): void {
+    if (this._isDisposed) {
+      return;
+    }
+    this._isDisposed = true;
+    this._contentManager.fileChanged.disconnect(this._onFileChanged);
+  }
+
   async flattenDirectory(
     dirContent: Contents.IModel
   ): Promise<IDict<{ code: string }>> {
@@ -138,4 +150,6 @@ export class SandpackFilesModel {
   private _contentManager: Contents.IManager;
 
   private _allFiles?: IDict<{ code: string }>;
+
+  private _isDisposed = false;
 }

package/src/sandpackWidget/sandpackPanel.ts

@@ -7,8 +7,9 @@ import { DocumentRegistry } from '@jupyterlab/docregistry';
 import { Contents } from '@jupyterlab/services';
 
 import { IFramePanel } from '../document/iframePanel';
-import { IDict } from '../type';
+import { IDict, IJupyterPackFileFormat } from '../type';
 import { SandpackFilesModel } from './sandpackFilesModel';
+import { PromiseDelegate } from '@lumino/coreutils';
 
 export class SandpackPanel extends IFramePanel {
   constructor(options: {
@@ -17,17 +18,44 @@ export class SandpackPanel extends IFramePanel {
   }) {
     super();
     this._contentsManager = options.contentsManager;
+    const { context } = options;
     options.context.ready.then(async () => {
-      await this.init(options.context.localPath);
+      const jpackModel =
+        context.model.toJSON() as any as IJupyterPackFileFormat;
+      await this.init(context.localPath, jpackModel);
     });
   }
 
-  async init(localPath: string) {
+  dispose(): void {
+    this._fileModel?.fileChanged.disconnect(this._onFileChanged);
+    this._fileModel?.dispose();
+    this._spClient?.destroy();
+    super.dispose();
+  }
+
+  get isReady() {
+    return this._isReady.promise;
+  }
+
+  get autoreload(): boolean {
+    return this._autoreload;
+  }
+
+  set autoreload(val: boolean) {
+    this._autoreload = val;
+  }
+
+  async init(localPath: string, jpackModel: IJupyterPackFileFormat) {
+    if (jpackModel?.metadata?.autoreload === true) {
+      this._autoreload = true;
+    }
+
     const currentDir = localPath.split('/').slice(0, -1).join('/');
     const filesModel = new SandpackFilesModel({
       contentsManager: this._contentsManager,
       path: currentDir
     });
+    this._fileModel = filesModel;
     const allFiles = await filesModel.getAllFiles();
 
     const options: ClientOptions = {
@@ -43,6 +71,7 @@ export class SandpackPanel extends IFramePanel {
       options
     );
     await this.connectSignals(filesModel, this._spClient);
+    this._isReady.resolve();
   }
 
   async connectSignals(
@@ -66,11 +95,20 @@ export class SandpackPanel extends IFramePanel {
     });
   }
 
+  async reload(): Promise<void> {
+    if (this._spClient && this._fileModel) {
+      const allFiles = await this._fileModel.getAllFiles();
+      this._spClient.updateSandbox({
+        files: allFiles
+      });
+    }
+  }
+
   private _onFileChanged(
     sender: SandpackFilesModel,
     { allFiles }: { allFiles: IDict<{ code: string }> }
   ) {
-    if (this._spClient) {
+    if (this._autoreload && this._spClient) {
       this._spClient.updateSandbox({
         files: allFiles
       });
@@ -79,4 +117,7 @@ export class SandpackPanel extends IFramePanel {
 
   private _spClient?: SandpackClient;
   private _contentsManager: Contents.IManager;
+  private _fileModel: SandpackFilesModel | undefined;
+  private _autoreload = false;
+  private _isReady = new PromiseDelegate<void>();
 }

package/src/swConnection/index.ts

@@ -8,7 +8,7 @@ import { expose } from 'comlink';
 
 import { IConnectionManagerToken } from '../token';
 import { IConnectionManager, MessageAction } from '../type';
-import { ConnectionManager } from './connection_manager';
+import { ConnectionManager } from './mainConnectionManager';
 
 const fullLabextensionsUrl = PageConfig.getOption('fullLabextensionsUrl');
 const SCOPE = `${fullLabextensionsUrl}/jupyterpack/static`;
@@ -66,7 +66,6 @@ export const swPlugin: JupyterFrontEndPlugin<IConnectionManager> = {
   autoStart: true,
   provides: IConnectionManagerToken,
   activate: async (app: JupyterFrontEnd): Promise<IConnectionManager> => {
-    console.log('Activating jupyterpack service worker');
     const serviceWorker = await initServiceWorker();
     if (!serviceWorker) {
       throw new Error(
@@ -75,6 +74,10 @@ export const swPlugin: JupyterFrontEndPlugin<IConnectionManager> = {
     }
 
     const instanceId = UUID.uuid4();
+    console.log(
+      'Activating jupyterpack service worker with instance id',
+      instanceId
+    );
     const { port1: mainToServiceWorker, port2: serviceWorkerToMain } =
       new MessageChannel();
 

package/src/swConnection/mainConnectionManager.ts

@@ -0,0 +1,121 @@
+import { arrayBufferToBase64 } from '../tools';
+import {
+  IBroadcastMessage,
+  IConnectionManager,
+  IDict,
+  IKernelExecutor
+} from '../type';
+import { UUID } from '@lumino/coreutils';
+
+/**
+ * Manages connections between clients and kernel executors.
+ * This class handles the registration of kernel executors and the generation of responses
+ * for client requests. It maintains a mapping of kernel client IDs to their respective executors.
+ * The HTTP requests intercepted by the service worker are forwarded to the appropriate kernel executor.
+ * The websocket messages forwarded from the broadcast channel are also forwarded to the appropriate kernel executor.
+ * It's running on the main thread
+ */
+export class ConnectionManager implements IConnectionManager {
+  constructor(public instanceId: string) {
+    this._wsBroadcastChannel = new BroadcastChannel(
+      `/jupyterpack/ws/${instanceId}`
+    );
+    this._initWsChannel();
+  }
+
+  async registerConnection(
+    kernelExecutor: IKernelExecutor
+  ): Promise<{ instanceId: string; kernelClientId: string }> {
+    const uuid = UUID.uuid4();
+
+    this._kernelExecutors.set(uuid, kernelExecutor);
+
+    return { instanceId: this.instanceId, kernelClientId: uuid };
+  }
+
+  async generateResponse(options: {
+    kernelClientId: string;
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    requestBody?: ArrayBuffer;
+    params?: string;
+  }): Promise<IDict | null> {
+    const { urlPath, kernelClientId, method, params, requestBody, headers } =
+      options;
+    const executor = this._kernelExecutors.get(kernelClientId);
+    if (!executor) {
+      return null;
+    }
+
+    const response = await executor.getResponse({
+      urlPath,
+      method,
+      params,
+      headers,
+      requestBody
+    });
+    return response;
+  }
+  private _initWsChannel() {
+    this._wsBroadcastChannel.onmessage = event => {
+      const rawData = event.data;
+      let data: IBroadcastMessage;
+      if (typeof rawData === 'string') {
+        data = JSON.parse(rawData);
+      } else {
+        data = rawData;
+      }
+
+      const { action, dest, wsUrl, payload } = data;
+      const executor = this._kernelExecutors.get(dest);
+      if (!executor) {
+        console.error(
+          'Missing kernel handle for message',
+          data,
+          dest,
+          this._kernelExecutors
+        );
+        return;
+      }
+
+      switch (action) {
+        case 'open': {
+          executor.openWebsocket({
+            instanceId: this.instanceId,
+            kernelId: dest,
+            wsUrl,
+            protocol: payload.protocol
+          });
+          break;
+        }
+        case 'send': {
+          let serializedData: string;
+          let isBinary: boolean;
+          if (payload instanceof ArrayBuffer || ArrayBuffer.isView(payload)) {
+            // Convert data to base64 string
+            serializedData = arrayBufferToBase64(payload as any);
+            isBinary = true;
+          } else if (typeof payload === 'string') {
+            serializedData = payload;
+            isBinary = false;
+          } else {
+            console.error('Unknown message type', payload);
+            return;
+          }
+          executor.sendWebsocketMessage({
+            instanceId: this.instanceId,
+            kernelId: dest,
+            wsUrl,
+            message: JSON.stringify({ isBinary, data: serializedData })
+          });
+          break;
+        }
+        default:
+          break;
+      }
+    };
+  }
+  private _kernelExecutors = new Map<string, IKernelExecutor>();
+  private _wsBroadcastChannel: BroadcastChannel;
+}

package/src/swConnection/sw.ts

@@ -1,6 +1,6 @@
 // import { expose } from 'comlink';
 import { MessageAction } from '../type';
-import { CommManager } from './comm_manager';
+import { CommManager } from './swCommManager';
 
 const COMM_MANAGER = new CommManager();
 /**