jupyterpack 0.2.1 → 0.4.0

package/src/document/plugin.ts

@@ -4,20 +4,29 @@ import {
 } from '@jupyterlab/application';
 
 import { JupyterPackWidgetFactory } from './widgetFactory';
-import { IConnectionManager } from '../type';
-import { IConnectionManagerToken } from '../token';
+import { IConnectionManager, IJupyterpackDocTracker } from '../type';
+import { IConnectionManagerToken, IJupyterpackDocTrackerToken } from '../token';
+import { WidgetTracker } from '@jupyterlab/apputils';
+import { DocumentWidget } from '@jupyterlab/docregistry';
+import { logoIcon } from '../tools';
+import { addCommands } from './commands';
 
 const FACTORY = 'jupyterpack';
 const CONTENT_TYPE = 'jupyterpack';
 
-export const spkPlugin: JupyterFrontEndPlugin<void> = {
+export const spkPlugin: JupyterFrontEndPlugin<IJupyterpackDocTracker> = {
   id: 'jupyterpack:spkplugin',
   requires: [IConnectionManagerToken],
   autoStart: true,
+  provides: IJupyterpackDocTrackerToken,
   activate: (
     app: JupyterFrontEnd,
     connectionManager: IConnectionManager
-  ): void => {
+  ): IJupyterpackDocTracker => {
+    const tracker = new WidgetTracker<DocumentWidget>({
+      namespace: FACTORY
+    });
+    addCommands(app.commands, tracker);
     const widgetFactory = new JupyterPackWidgetFactory({
       name: FACTORY,
       modelName: 'text',
@@ -25,7 +34,8 @@ export const spkPlugin: JupyterFrontEndPlugin<void> = {
       defaultFor: [CONTENT_TYPE],
       commands: app.commands,
       manager: app.serviceManager,
-      connectionManager
+      connectionManager,
+      tracker
     });
 
     // Registering the widget factory
@@ -35,10 +45,21 @@ export const spkPlugin: JupyterFrontEndPlugin<void> = {
     app.docRegistry.addFileType({
       name: CONTENT_TYPE,
       displayName: 'SPK',
-      mimeTypes: ['text/json'],
+      mimeTypes: ['application/json'],
       extensions: ['.spk', '.SPK'],
       fileFormat: 'json',
-      contentType: CONTENT_TYPE
+      contentType: CONTENT_TYPE,
+      icon: logoIcon
+    });
+
+    widgetFactory.widgetCreated.connect((_, widget) => {
+      widget.title.icon = logoIcon;
+      widget.context.pathChanged.connect(() => {
+        tracker.save(widget);
+      });
+      tracker.add(widget);
     });
+
+    return tracker;
   }
 };

package/src/document/toolbar.ts

@@ -0,0 +1,39 @@
+import {
+  CommandToolbarButton,
+  ReactiveToolbar
+} from '@jupyterlab/ui-components';
+import { CommandRegistry } from '@lumino/commands';
+
+import { IJupyterpackDocTracker } from '../type';
+import { CommandIDs } from './commands';
+
+export class ToolbarWidget extends ReactiveToolbar {
+  constructor(options: {
+    tracker: IJupyterpackDocTracker;
+    commands: CommandRegistry;
+  }) {
+    super();
+    this.addClass('jupyterpack-toolbar');
+    this.addItem(
+      'Reload',
+      new CommandToolbarButton({
+        commands: options.commands,
+        id: CommandIDs.RELOAD
+      })
+    );
+    this.addItem(
+      'Toggle Auto Reload',
+      new CommandToolbarButton({
+        id: CommandIDs.TOGGLE_AUTORELOAD,
+        commands: options.commands
+      })
+    );
+    this.addItem(
+      'Open Specta',
+      new CommandToolbarButton({
+        id: CommandIDs.OPEN_SPECTA,
+        commands: options.commands
+      })
+    );
+  }
+}

package/src/document/widgetFactory.ts

@@ -1,23 +1,26 @@
 import { ABCWidgetFactory, DocumentRegistry } from '@jupyterlab/docregistry';
 import { Contents, ServiceManager } from '@jupyterlab/services';
 import { CommandRegistry } from '@lumino/commands';
+import { UUID } from '@lumino/coreutils';
 import { Panel } from '@lumino/widgets';
 
+import { PythonWidget } from '../pythonWidget/pythonWidget';
+import { PythonWidgetModel } from '../pythonWidget/pythonWidgetModel';
+import { SandpackPanel } from '../sandpackWidget/sandpackPanel';
 import {
   IConnectionManager,
+  IJupyterpackDocTracker,
   IJupyterPackFileFormat,
   JupyterPackFramework
 } from '../type';
-import { SandpackPanel } from '../sandpackWidget/sandpackPanel';
 import { JupyterPackDocWidget } from './jupyterpackDocWidget';
-import { PythonWidgetModel } from '../pythonWidget/pythonWidgetModel';
-import { UUID } from '@lumino/coreutils';
-import { PythonWidget } from '../pythonWidget/pythonWidget';
+import { ToolbarWidget } from './toolbar';
 
 interface IOptions extends DocumentRegistry.IWidgetFactoryOptions {
   commands: CommandRegistry;
   manager: ServiceManager.IManager;
   connectionManager: IConnectionManager;
+  tracker: IJupyterpackDocTracker;
 }
 
 export class JupyterPackWidgetFactory extends ABCWidgetFactory<JupyterPackDocWidget> {
@@ -49,8 +52,11 @@ export class JupyterPackWidgetFactory extends ABCWidgetFactory<JupyterPackDocWid
           content.addWidget(jpContent);
           break;
         }
-        case JupyterPackFramework.DASH: {
+        case JupyterPackFramework.DASH:
+        case JupyterPackFramework.STREAMLIT:
+        case JupyterPackFramework.TORNADO: {
           const model = new PythonWidgetModel({
+            jpackModel,
             context,
             manager: this.options.manager,
             contentsManager: this._contentsManager,
@@ -69,9 +75,14 @@ export class JupyterPackWidgetFactory extends ABCWidgetFactory<JupyterPackDocWid
         }
       }
     });
+    const toolbar = new ToolbarWidget({
+      tracker: this.options.tracker,
+      commands: this.options.commands
+    });
     return new JupyterPackDocWidget({
       context,
-      content
+      content,
+      toolbar
     });
   }
 

package/src/global.d.ts

@@ -0,0 +1,9 @@
+declare module '*?raw' {
+  const content: string;
+  export default content;
+}
+
+declare module '*.svg' {
+  const value: string;
+  export default value;
+}

package/src/pythonServer/dash/dashServer.ts

@@ -0,0 +1,66 @@
+import { stringOrNone } from '../../tools';
+import { IDict, JupyterPackFramework } from '../../type';
+import { KernelExecutor } from '../kernelExecutor';
+
+export class DashServer extends KernelExecutor {
+  async init(options: {
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }) {
+    await super.init(options);
+    const { initCode, instanceId, kernelClientId } = options;
+
+    const baseURL = this.buildBaseURL({
+      instanceId,
+      kernelClientId,
+      framework: JupyterPackFramework.DASH
+    });
+    await this.executeCode({
+      code: `
+      import os
+      os.environ["DASH_URL_BASE_PATHNAME"] = "${baseURL}"
+      `
+    });
+    if (initCode) {
+      await this.executeCode({ code: initCode });
+    }
+    const loaderCode = `
+      from jupyterpack.dash import DashServer
+      ${this._DASH_SERVER_VAR} = DashServer(app, "${baseURL}")
+      `;
+    await this.executeCode({ code: loaderCode });
+  }
+
+  getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }) {
+    const { method, urlPath, headers, params, content } = options;
+    const code = `${this._DASH_SERVER_VAR}.get_response("${method}", "${urlPath}", headers=${JSON.stringify(headers)} , content=${stringOrNone(content)}, params=${stringOrNone(params)})`;
+    return code;
+  }
+
+  async disposePythonServer(): Promise<void> {
+    await this.executeCode({ code: `${this._DASH_SERVER_VAR}.dispose()` });
+  }
+
+  async reloadPythonServer(options: {
+    entryPath?: string;
+    initCode?: string;
+  }): Promise<void> {
+    const { initCode } = options;
+    if (initCode) {
+      await this.executeCode({ code: initCode });
+    }
+    await this.executeCode(
+      { code: `${this._DASH_SERVER_VAR}.reload(app)` },
+      true
+    );
+  }
+
+  private _DASH_SERVER_VAR = '__jupyterpack_dash_server';
+}

package/src/pythonServer/index.ts

@@ -0,0 +1,18 @@
+import { IKernelExecutor, JupyterPackFramework } from '../type';
+import { DashServer } from './dash/dashServer';
+import { KernelExecutor } from './kernelExecutor';
+import { StreamlitServer } from './streamlit/streamlitServer';
+import { TornadoServer } from './tornado/tornadoServer';
+
+type KernelExecutorConstructor = new (
+  options: KernelExecutor.IOptions
+) => IKernelExecutor;
+
+export const PYTHON_SERVER = new Map<
+  JupyterPackFramework,
+  KernelExecutorConstructor
+>([
+  [JupyterPackFramework.DASH, DashServer],
+  [JupyterPackFramework.STREAMLIT, StreamlitServer],
+  [JupyterPackFramework.TORNADO, TornadoServer]
+]);

package/src/pythonServer/kernelExecutor.ts

@@ -0,0 +1,243 @@
+import { PageConfig, URLExt } from '@jupyterlab/coreutils';
+import { KernelMessage, Session } from '@jupyterlab/services';
+import stripAnsi from 'strip-ansi';
+import {
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+  base64ToString,
+  isBinaryContentType
+} from '../tools';
+import { IDict, IKernelExecutor, JupyterPackFramework } from '../type';
+import websocketPatch from '../websocket/websocket.js?raw';
+
+export abstract class KernelExecutor implements IKernelExecutor {
+  constructor(options: KernelExecutor.IOptions) {
+    this._sessionConnection = options.sessionConnection;
+    this._wsPatch = websocketPatch.replaceAll('"use strict";', '');
+  }
+
+  get isDisposed(): boolean {
+    return this._isDisposed;
+  }
+
+  abstract disposePythonServer(): Promise<void>;
+  abstract reloadPythonServer(options: {
+    entryPath?: string;
+    initCode?: string;
+  }): Promise<void>;
+
+  abstract getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }): string;
+
+  async init(options: {
+    entryPath?: string;
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }): Promise<void> {
+    const patchCode = `
+    from jupyterpack.common import patch_all
+    patch_all()
+    `;
+    await this.executeCode({ code: patchCode });
+  }
+
+  openWebsocketFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): string | undefined {
+    return undefined;
+  }
+
+  sendWebsocketMessageFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): string | undefined {
+    return undefined;
+  }
+
+  async openWebsocket(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): Promise<void> {
+    const code = this.openWebsocketFunctionFactory(options);
+    if (code) {
+      await this.executeCode({ code });
+    }
+  }
+
+  async sendWebsocketMessage(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): Promise<void> {
+    const code = this.sendWebsocketMessageFunctionFactory(options);
+    if (code) {
+      await this.executeCode({ code });
+    }
+  }
+
+  async getResponse(options: {
+    method: string;
+    urlPath: string;
+    headers: IDict;
+    requestBody?: ArrayBuffer;
+    params?: string;
+  }): Promise<IDict> {
+    const { method, urlPath, requestBody, params, headers } = options;
+    const content = requestBody ? arrayBufferToBase64(requestBody) : undefined;
+    const code = this.getResponseFunctionFactory({
+      method,
+      urlPath,
+      headers,
+      params,
+      content
+    });
+    const raw = await this.executeCode({ code }, true);
+    if (!raw) {
+      throw new Error(`Missing response for ${urlPath}`);
+    }
+    const jsonStr = raw.replaceAll("'", '');
+    const obj: {
+      headers: string;
+      status_code: number;
+      content: string;
+    } = JSON.parse(jsonStr);
+    const responseHeaders: IDict<string> = JSON.parse(atob(obj.headers));
+    const contentType: string | undefined =
+      responseHeaders?.['Content-Type'] ?? responseHeaders?.['content-type'];
+    let responseContent: string | Uint8Array;
+
+    if (isBinaryContentType(contentType)) {
+      responseContent = base64ToArrayBuffer(obj.content);
+    } else {
+      responseContent = base64ToString(obj.content);
+    }
+
+    if (contentType && contentType.toLowerCase() === 'text/html') {
+      responseContent = (responseContent as string).replace(
+        '<head>',
+        `<head>\n<script>\n${this._wsPatch}\n</script>\n`
+      );
+    }
+
+    const decodedObj = {
+      status_code: obj.status_code,
+      headers: responseHeaders,
+      content: responseContent
+    };
+
+    return decodedObj;
+  }
+  async executeCode(
+    code: KernelMessage.IExecuteRequestMsg['content'],
+    waitForResult?: boolean
+  ): Promise<string | null> {
+    const kernel = this._sessionConnection?.kernel;
+    if (!kernel) {
+      throw new Error('Session has no kernel.');
+    }
+    return new Promise<string | null>((resolve, reject) => {
+      const future = kernel.requestExecute(code, false, undefined);
+      let executeResult = '';
+      future.onIOPub = (msg: KernelMessage.IIOPubMessage): void => {
+        const msgType = msg.header.msg_type;
+
+        switch (msgType) {
+          case 'execute_result': {
+            if (waitForResult) {
+              const content = (msg as KernelMessage.IExecuteResultMsg).content
+                .data['text/plain'] as string;
+              executeResult += content;
+              resolve(executeResult);
+            }
+            break;
+          }
+          case 'stream': {
+            const content = (msg as KernelMessage.IStreamMsg).content;
+            if (content.text.length === 0) {
+              break;
+            }
+            if (content.name === 'stderr') {
+              console.error('Kernel stream:', content.text);
+            } else {
+              console.log('Kernel stream:', content.text);
+            }
+            break;
+          }
+          case 'error': {
+            console.error(
+              'Kernel operation failed',
+              code.code,
+              (msg.content as any).traceback
+                .map((it: string) => stripAnsi(it))
+                .join('\n')
+            );
+
+            reject(msg.content);
+            break;
+          }
+          default:
+            break;
+        }
+      };
+      if (!waitForResult) {
+        resolve(null);
+        // future.dispose() # TODO
+      }
+    });
+  }
+
+  dispose(): void {
+    if (this._isDisposed) {
+      return;
+    }
+    this._isDisposed = true;
+    this._sessionConnection.dispose();
+  }
+
+  protected buildBaseURL(options: {
+    instanceId: string;
+    kernelClientId: string;
+    framework: JupyterPackFramework;
+  }) {
+    const { instanceId, kernelClientId, framework } = options;
+    const fullLabextensionsUrl = PageConfig.getOption('fullLabextensionsUrl');
+
+    const baseURL = URLExt.join(
+      fullLabextensionsUrl,
+      'jupyterpack/static',
+      instanceId,
+      framework,
+      kernelClientId,
+      '/'
+    );
+    this._baseUrl = baseURL;
+
+    return baseURL;
+  }
+
+  protected _baseUrl: string | undefined;
+
+  private _isDisposed: boolean = false;
+  private _sessionConnection: Session.ISessionConnection;
+  private _wsPatch: string;
+}
+
+export namespace KernelExecutor {
+  export interface IOptions {
+    sessionConnection: Session.ISessionConnection;
+  }
+}

package/src/pythonServer/streamlit/streamlitServer.ts

@@ -0,0 +1,67 @@
+import { JupyterPackFramework } from '../../type';
+import { TornadoServer } from '../tornado/tornadoServer';
+
+export class StreamlitServer extends TornadoServer {
+  async init(options: {
+    entryPath?: string;
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }) {
+    const { instanceId, kernelClientId, entryPath } = options;
+    if (!entryPath) {
+      throw new Error(
+        'Missing streamlit entry path, please check your SPK file'
+      );
+    }
+    const baseURL = this.buildBaseURL({
+      instanceId,
+      kernelClientId,
+      framework: JupyterPackFramework.STREAMLIT
+    });
+
+    const patchCode = `
+    from jupyterpack.common import set_base_url_env, patch_tornado, patch_all
+    patch_all()
+    patch_tornado()
+    set_base_url_env("${baseURL}")
+    `;
+    await this.executeCode({ code: patchCode });
+
+    const bootstrapCode = `
+    from jupyterpack.streamlit import patch_streamlit
+    patch_streamlit()
+    `;
+    await this.executeCode({ code: bootstrapCode });
+
+    const stCode = `
+      from jupyterpack.streamlit import StreamlitServer, create_streamlit_app
+      __jupyterpack_st_server, __jupyterpack_tor_app = await create_streamlit_app("${entryPath}", "${baseURL}")
+      ${this._SERVER_VAR} = StreamlitServer(__jupyterpack_tor_app, "${baseURL}", __jupyterpack_st_server)
+      `;
+    await this.executeCode({ code: stCode });
+  }
+
+  async reloadPythonServer(options: {
+    entryPath?: string;
+    initCode?: string;
+  }): Promise<void> {
+    const { entryPath } = options;
+    if (!entryPath || !this._baseUrl) {
+      return;
+    }
+    const reloadCode = `
+      ${this._SERVER_VAR}.dispose()
+      __jupyterpack_st_server, __jupyterpack_tor_app = await create_streamlit_app("${entryPath}", "${this._baseUrl}")
+      ${this._SERVER_VAR}.reload(__jupyterpack_tor_app, __jupyterpack_st_server)
+      `;
+    await this.executeCode(
+      {
+        code: reloadCode
+      },
+      true
+    );
+  }
+
+  protected _SERVER_VAR = '__jupyterpack_streamlit_server';
+}

package/src/pythonServer/tornado/tornadoServer.ts

@@ -0,0 +1,97 @@
+import { stringOrNone } from '../../tools';
+import { IDict, JupyterPackFramework } from '../../type';
+import { KernelExecutor } from '../kernelExecutor';
+
+export class TornadoServer extends KernelExecutor {
+  async init(options: {
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }) {
+    await super.init(options);
+    const { initCode, instanceId, kernelClientId } = options;
+
+    const baseURL = this.buildBaseURL({
+      instanceId,
+      kernelClientId,
+      framework: JupyterPackFramework.TORNADO
+    });
+    const bootstrapCode = `
+    from jupyterpack.common import set_base_url_env, patch_tornado
+    set_base_url_env("${baseURL}")
+    patch_tornado()
+
+    `;
+    await this.executeCode({ code: bootstrapCode });
+    if (initCode) {
+      const initCodeWithUrl = initCode.replaceAll('{{base_url}}', baseURL);
+      await this.executeCode({ code: initCodeWithUrl });
+      const loaderCode = `
+      from jupyterpack.tornado import TornadoServer
+      ${this._SERVER_VAR} = TornadoServer(app, "${baseURL}")
+      `;
+
+      await this.executeCode({ code: loaderCode });
+    }
+  }
+
+  getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }) {
+    const { method, urlPath, headers, params, content } = options;
+    const code = `await ${this._SERVER_VAR}.get_response("${method}", "${urlPath}", headers=${JSON.stringify(headers)} , content=${stringOrNone(content)}, params=${stringOrNone(params)})`;
+    return code;
+  }
+
+  openWebsocketFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): string {
+    const { instanceId, kernelId, wsUrl, protocol } = options;
+
+    const code = `await ${this._SERVER_VAR}.open_ws("${instanceId}", "${kernelId}", "${wsUrl}", ${stringOrNone(protocol)})`;
+    return code;
+  }
+
+  sendWebsocketMessageFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): string {
+    const { instanceId, kernelId, wsUrl, message } = options;
+    const code = `await ${this._SERVER_VAR}.receive_ws_message("${instanceId}", "${kernelId}", "${wsUrl}", '''${message}''')`;
+    return code;
+  }
+
+  async disposePythonServer(): Promise<void> {
+    await this.executeCode({
+      code: `${this._SERVER_VAR}.dispose()`
+    });
+  }
+
+  async reloadPythonServer(options: {
+    entryPath?: string;
+    initCode?: string;
+  }): Promise<void> {
+    const { initCode } = options;
+    if (initCode) {
+      await this.executeCode({
+        code: initCode.replaceAll('{{base_url}}', this._baseUrl ?? '')
+      });
+      const reloadCode = `
+      ${this._SERVER_VAR}.dispose()
+      ${this._SERVER_VAR}.reload(app)
+      `;
+      await this.executeCode({ code: reloadCode }, true);
+    }
+  }
+
+  protected _SERVER_VAR = '__jupyterpack_tornado_server';
+}