jupyterpack 0.2.0 → 0.3.0

package/src/pythonServer/common/generatedPythonFiles.ts

@@ -0,0 +1,73 @@
+// Auto-generated TypeScript file from Python files
+
+export const patch = `
+import pyodide_http
+import collections
+
+if not hasattr(collections, "MutableSet"):
+    import collections.abc
+
+    collections.MutableSet = collections.abc.MutableSet
+
+pyodide_http.patch_all()`;
+export const tools = `
+import importlib.util
+import sys
+from types import ModuleType
+
+import tempfile
+from pathlib import Path
+from typing import List
+import os
+
+os.environ.setdefault("JUPYTERPACK_BASE_URL", "{{base_url}}")
+
+
+def __jupyterpack_import_from_path(module_name: str, path: str) -> ModuleType:
+    """
+    Import a Python module from a given file path.
+    Always reloads (does not use sys.modules cache).
+    """
+    # Remove from sys.modules if already loaded
+    if module_name in sys.modules:
+        del sys.modules[module_name]
+
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot import module {module_name} from {path}")
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def __jupyterpack_create_mock_module(
+    module_names: List[str], mock_content: str, patch_parent=True
+):
+    tmpdir = tempfile.TemporaryDirectory()
+    package_dir = Path(tmpdir.name) / "__jupyterpack_mock_module"
+    package_dir.mkdir()
+    (package_dir / "__init__.py").write_text(mock_content)
+
+    sys.path.insert(0, tmpdir.name)
+    mock_module = importlib.import_module("__jupyterpack_mock_module")
+    sys.path.pop(0)
+    for module_name in module_names:
+        if patch_parent:
+            parts = module_name.split(".")
+            for i in range(1, len(parts) + 1):
+                subpath = ".".join(parts[:i])
+                sys.modules[subpath] = mock_module
+
+            for i in range(1, len(parts)):
+                parent_name = ".".join(parts[:i])
+                child_name = ".".join(parts[: i + 1])
+                parent_mod = sys.modules[parent_name]
+                child_mod = sys.modules[child_name]
+                setattr(parent_mod, parts[i], child_mod)
+        else:
+            sys.modules[module_name] = mock_module
+
+    return tmpdir
+`;

package/src/pythonServer/dash/dashServer.ts

@@ -0,0 +1,57 @@
+import { stringOrNone } from '../../tools';
+import { IDict, JupyterPackFramework } from '../../type';
+import { patch } from '../common/generatedPythonFiles';
+import { KernelExecutor } from '../kernelExecutor';
+import { bootstrap, dashLoader } from './generatedPythonFiles';
+
+export class DashServer extends KernelExecutor {
+  async init(options: {
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }) {
+    await super.init(options);
+    const { initCode, instanceId, kernelClientId } = options;
+
+    const baseURL = this.buildBaseURL({
+      instanceId,
+      kernelClientId,
+      framework: JupyterPackFramework.DASH
+    });
+    await this.executeCode({ code: patch });
+    await this.executeCode({
+      code: bootstrap.replaceAll('{{base_url}}', baseURL)
+    });
+    if (initCode) {
+      await this.executeCode({ code: initCode });
+    }
+    await this.executeCode({ code: dashLoader });
+  }
+
+  getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }) {
+    const { method, urlPath, headers, params, content } = options;
+    const code = `${this.DASH_GET_RESPONSE_FUNCTION}("${method}", "${urlPath}", headers=${JSON.stringify(headers)} , content=${stringOrNone(content)}, params=${stringOrNone(params)})`;
+    return code;
+  }
+
+  async disposePythonServer(): Promise<void> {
+    //no-op
+  }
+
+  async openWebsocket(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): Promise<void> {
+    //no-op
+  }
+
+  private DASH_GET_RESPONSE_FUNCTION = '__jupyterpack_dash_get_response';
+}

package/src/pythonServer/dash/generatedPythonFiles.ts

@@ -0,0 +1,32 @@
+// Auto-generated TypeScript file from Python files
+
+export const bootstrap = `
+import os
+
+os.environ["DASH_URL_BASE_PATHNAME"] = "{{base_url}}"
+`;
+export const dashLoader = `
+import httpx, json, base64
+__jupyterpack_dash_transport = httpx.WSGITransport(app=app.server)  # noqa
+
+
+def __jupyterpack_dash_get_response(method, url, headers, content=None, params=None):
+    decoded_content = None
+    if content is not None:
+        decoded_content = base64.b64decode(content)
+        # decoded_content = content.decode()
+    with httpx.Client(
+        transport=__jupyterpack_dash_transport, base_url="http://testserver"
+    ) as client:
+        r = client.request(
+            method, url, headers=headers, content=decoded_content, params=params
+        )
+        reply_headers = json.dumps(dict(r.headers)).encode("utf-8")
+        response = {
+            "headers": base64.b64encode(reply_headers).decode("ascii"),
+            "content": base64.b64encode(r.content).decode("ascii"),
+            "status_code": r.status_code,
+        }
+        json_str = json.dumps(response)
+        return json_str
+`;

package/src/pythonServer/index.ts

@@ -0,0 +1,18 @@
+import { IKernelExecutor, JupyterPackFramework } from '../type';
+import { DashServer } from './dash/dashServer';
+import { KernelExecutor } from './kernelExecutor';
+import { StreamlitServer } from './streamlit/streamlitServer';
+import { TornadoServer } from './tornado/tornadoServer';
+
+type KernelExecutorConstructor = new (
+  options: KernelExecutor.IOptions
+) => IKernelExecutor;
+
+export const PYTHON_SERVER = new Map<
+  JupyterPackFramework,
+  KernelExecutorConstructor
+>([
+  [JupyterPackFramework.DASH, DashServer],
+  [JupyterPackFramework.STREAMLIT, StreamlitServer],
+  [JupyterPackFramework.TORNADO, TornadoServer]
+]);

package/src/pythonServer/kernelExecutor.ts

@@ -0,0 +1,229 @@
+import { PageConfig, URLExt } from '@jupyterlab/coreutils';
+import { KernelMessage, Session } from '@jupyterlab/services';
+import stripAnsi from 'strip-ansi';
+import {
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+  base64ToString,
+  isBinaryContentType
+} from '../tools';
+import { IDict, IKernelExecutor, JupyterPackFramework } from '../type';
+import websocketPatch from '../websocket/websocket.js?raw';
+import { patch } from './common/generatedPythonFiles';
+
+export abstract class KernelExecutor implements IKernelExecutor {
+  constructor(options: KernelExecutor.IOptions) {
+    this._sessionConnection = options.sessionConnection;
+    this._wsPatch = websocketPatch.replaceAll('"use strict";', '');
+  }
+
+  get isDisposed(): boolean {
+    return this._isDisposed;
+  }
+
+  abstract disposePythonServer(): Promise<void>;
+
+  abstract getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }): string;
+
+  async init(options: {
+    entryPath?: string;
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }): Promise<void> {
+    await this.executeCode({ code: patch });
+  }
+
+  openWebsocketFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): string | undefined {
+    return undefined;
+  }
+
+  sendWebsocketMessageFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): string | undefined {
+    return undefined;
+  }
+
+  async openWebsocket(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): Promise<void> {
+    const code = this.openWebsocketFunctionFactory(options);
+    if (code) {
+      await this.executeCode({ code });
+    }
+  }
+
+  async sendWebsocketMessage(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): Promise<void> {
+    const code = this.sendWebsocketMessageFunctionFactory(options);
+    if (code) {
+      await this.executeCode({ code });
+    }
+  }
+
+  async getResponse(options: {
+    method: string;
+    urlPath: string;
+    headers: IDict;
+    requestBody?: ArrayBuffer;
+    params?: string;
+  }): Promise<IDict> {
+    const { method, urlPath, requestBody, params, headers } = options;
+    const content = requestBody ? arrayBufferToBase64(requestBody) : undefined;
+    const code = this.getResponseFunctionFactory({
+      method,
+      urlPath,
+      headers,
+      params,
+      content
+    });
+    const raw = await this.executeCode({ code }, true);
+    if (!raw) {
+      throw new Error(`Missing response for ${urlPath}`);
+    }
+    const jsonStr = raw.replaceAll("'", '');
+    const obj: {
+      headers: string;
+      status_code: number;
+      content: string;
+    } = JSON.parse(jsonStr);
+    const responseHeaders: IDict<string> = JSON.parse(atob(obj.headers));
+    const contentType: string | undefined =
+      responseHeaders?.['Content-Type'] ?? responseHeaders?.['content-type'];
+    let responseContent: string | Uint8Array;
+
+    if (isBinaryContentType(contentType)) {
+      responseContent = base64ToArrayBuffer(obj.content);
+    } else {
+      responseContent = base64ToString(obj.content);
+    }
+
+    if (contentType && contentType.toLowerCase() === 'text/html') {
+      responseContent = (responseContent as string).replace(
+        '<head>',
+        `<head>\n<script>\n${this._wsPatch}\n</script>\n`
+      );
+    }
+
+    const decodedObj = {
+      status_code: obj.status_code,
+      headers: responseHeaders,
+      content: responseContent
+    };
+
+    return decodedObj;
+  }
+  async executeCode(
+    code: KernelMessage.IExecuteRequestMsg['content'],
+    waitForResult?: boolean
+  ): Promise<string | null> {
+    const kernel = this._sessionConnection?.kernel;
+    if (!kernel) {
+      throw new Error('Session has no kernel.');
+    }
+    return new Promise<string | null>((resolve, reject) => {
+      const future = kernel.requestExecute(code, false, undefined);
+      let executeResult = '';
+      future.onIOPub = (msg: KernelMessage.IIOPubMessage): void => {
+        const msgType = msg.header.msg_type;
+
+        switch (msgType) {
+          case 'execute_result': {
+            if (waitForResult) {
+              const content = (msg as KernelMessage.IExecuteResultMsg).content
+                .data['text/plain'] as string;
+              executeResult += content;
+              resolve(executeResult);
+            }
+            break;
+          }
+          case 'stream': {
+            const content = (msg as KernelMessage.IStreamMsg).content;
+            if (content.name === 'stderr') {
+              console.error('Kernel stream', content.text);
+            } else {
+              console.log('Kernel stream', content.text);
+            }
+            break;
+          }
+          case 'error': {
+            console.error(
+              'Kernel operation failed',
+              code.code,
+              (msg.content as any).traceback
+                .map((it: string) => stripAnsi(it))
+                .join('\n')
+            );
+
+            reject(msg.content);
+            break;
+          }
+          default:
+            break;
+        }
+      };
+      if (!waitForResult) {
+        resolve(null);
+        // future.dispose() # TODO
+      }
+    });
+  }
+
+  dispose(): void {
+    if (this._isDisposed) {
+      return;
+    }
+    this._isDisposed = true;
+    this._sessionConnection.dispose();
+  }
+
+  protected buildBaseURL(options: {
+    instanceId: string;
+    kernelClientId: string;
+    framework: JupyterPackFramework;
+  }) {
+    const { instanceId, kernelClientId, framework } = options;
+    const labBaseUrl = PageConfig.getOption('baseUrl');
+    const baseURL = URLExt.join(
+      labBaseUrl,
+      'extensions/jupyterpack/static',
+      instanceId,
+      framework,
+      kernelClientId,
+      '/'
+    );
+
+    return baseURL;
+  }
+
+  private _isDisposed: boolean = false;
+  private _sessionConnection: Session.ISessionConnection;
+  private _wsPatch: string;
+}
+
+export namespace KernelExecutor {
+  export interface IOptions {
+    sessionConnection: Session.ISessionConnection;
+  }
+}

package/src/pythonServer/streamlit/generatedPythonFiles.ts

@@ -0,0 +1,148 @@
+// Auto-generated TypeScript file from Python files
+
+export const bootstrap = `
+import os
+
+import threading
+import streamlit.watcher.path_watcher
+import contextlib
+import streamlit.elements.spinner
+
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+
+class MockedThread(threading.Thread):
+    def start(self):
+        threading.current_thread = lambda: self
+        try:
+            self.run()
+        except Exception as e:
+            raise e
+
+
+threading.Thread = MockedThread
+
+
+class WatcherMock:
+    def __init__(
+        self,
+        path,
+        callback,
+        glob_pattern: str | None = None,
+        allow_nonexistent: bool = False,
+    ) -> None:
+        pass
+
+    def close(self) -> None:
+        pass
+
+
+streamlit.watcher.path_watcher.watchdog_available = False
+streamlit.watcher.path_watcher.EventBasedPathWatcher = WatcherMock
+streamlit.watcher.path_watcher._is_watchdog_available = lambda: False
+streamlit.watcher.path_watcher.get_path_watcher_class = lambda x: WatcherMock
+
+
+class MockThreading:
+    class Timer:
+        def __init__(self, delay, cb):
+            cb()
+
+        def start(self):
+            pass
+
+    Lock = contextlib.nullcontext
+
+
+streamlit.elements.spinner.threading = MockThreading
+`;
+export const streamlitLoader = `
+import json
+from streamlit import config
+import streamlit.web.server.server as st_server
+from streamlit.runtime.runtime import Runtime
+
+try:
+    # Check if __jupyterpack_streamlit_instance defined from previous run
+    __jupyterpack_streamlit_instance
+except NameError:
+    __jupyterpack_streamlit_instance = {
+        "tornado_bridge": None,
+        "streamlit_server": None,
+    }
+
+
+def __jupyterpack_create_streamlit_app(base_url, script_path):
+    if Runtime._instance is not None:
+        Runtime._instance.stop()
+        Runtime._instance = None
+    config.set_option("server.baseUrlPath", base_url)
+
+    config.set_option("server.port", 6789)
+    config.set_option("server.enableCORS", False)
+    config.set_option("server.enableXsrfProtection", False)
+
+    streamlit_server = st_server.Server(script_path, True)
+    return streamlit_server
+
+
+def __jupyterpack_streamlit_dispose():
+    global __jupyterpack_streamlit_instance
+    streamlit_server = __jupyterpack_streamlit_instance.get("streamlit_server", None)
+    if streamlit_server:
+        streamlit_server._runtime.stop()
+
+    __jupyterpack_streamlit_instance = {
+        "tornado_bridge": None,
+        "streamlit_server": None,
+    }
+    del streamlit_server
+
+
+async def __jupyterpack_streamlit_open_ws(
+    instance_id: str, kernel_client_id: str, ws_url: str, protocols_str: str | None
+):
+    tornado_bridge = __jupyterpack_streamlit_instance["tornado_bridge"]
+    if tornado_bridge is None:
+        raise Exception("Missing tornado instance")
+    await tornado_bridge.open_ws(instance_id, kernel_client_id, ws_url, protocols_str)
+
+
+async def __jupyterpack_streamlit_receive_ws_message(
+    instance_id: str, kernel_client_id: str, ws_url: str, payload_message: str
+):
+    tornado_bridge = __jupyterpack_streamlit_instance["tornado_bridge"]
+    if tornado_bridge is None:
+        raise Exception("Missing tornado instance")
+    await tornado_bridge.receive_ws_message_from_js(
+        instance_id, kernel_client_id, ws_url, payload_message
+    )
+
+
+async def __jupyterpack_streamlit_get_response(
+    method, url, headers, content=None, params=None
+):
+    global __jupyterpack_streamlit_instance
+    if not __jupyterpack_streamlit_instance["streamlit_server"]:
+        streamlit_server = __jupyterpack_create_streamlit_app(
+            "{{base_url}}", "{{script_path}}"
+        )  # noqa
+        app = streamlit_server._create_app()
+        await streamlit_server._runtime.start()
+        __jupyterpack_streamlit_instance["streamlit_server"] = streamlit_server
+        __jupyterpack_streamlit_instance["tornado_bridge"] = TornadoBridge(
+            app, "{{base_url}}"
+        )
+
+    tornado_bridge = __jupyterpack_streamlit_instance["tornado_bridge"]
+    req_dict = {
+        "method": method,
+        "url": url,
+        "headers": list(headers.items()),
+        "body": content,
+    }
+
+    response = await tornado_bridge.fetch(req_dict)
+    json_str = json.dumps(response)
+    return json_str
+`;

package/src/pythonServer/streamlit/streamlitServer.ts

@@ -0,0 +1,87 @@
+import { stringOrNone } from '../../tools';
+import { IDict, JupyterPackFramework } from '../../type';
+import { patch, tools } from '../common/generatedPythonFiles';
+import { KernelExecutor } from '../kernelExecutor';
+import {
+  bootstrap as tornadoBootstrap,
+  tornadoBridge
+} from '../tornado/generatedPythonFiles';
+import { bootstrap, streamlitLoader } from './generatedPythonFiles';
+
+export class StreamlitServer extends KernelExecutor {
+  async init(options: {
+    entryPath?: string;
+    initCode?: string;
+    instanceId: string;
+    kernelClientId: string;
+  }) {
+    await super.init(options);
+    const { instanceId, kernelClientId, entryPath } = options;
+    if (!entryPath) {
+      throw new Error(
+        'Missing streamlit entry path, please check your SPK file'
+      );
+    }
+    const baseURL = this.buildBaseURL({
+      instanceId,
+      kernelClientId,
+      framework: JupyterPackFramework.STREAMLIT
+    });
+    await this.executeCode({ code: patch });
+    await this.executeCode({ code: tools.replaceAll('{{base_url}}', baseURL) });
+    await this.executeCode({ code: tornadoBootstrap });
+    await this.executeCode({ code: tornadoBridge });
+    await this.executeCode({ code: bootstrap });
+
+    const stCode = streamlitLoader
+      .replaceAll('{{base_url}}', baseURL)
+      .replaceAll('{{script_path}}', entryPath);
+    await this.executeCode({ code: stCode });
+  }
+
+  getResponseFunctionFactory(options: {
+    urlPath: string;
+    method: string;
+    headers: IDict;
+    params?: string;
+    content?: string;
+  }) {
+    const { method, urlPath, headers, params, content } = options;
+    const code = `await ${this._GET_RESPONSE_FUNCTION}("${method}", "${urlPath}", headers=${JSON.stringify(headers)} , content=${stringOrNone(content)}, params=${stringOrNone(params)})`;
+    return code;
+  }
+
+  openWebsocketFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    protocol?: string;
+  }): string {
+    const { instanceId, kernelId, wsUrl, protocol } = options;
+
+    const code = `await ${this._OPEN_WEBSOCKET_FUNCTION}("${instanceId}", "${kernelId}", "${wsUrl}", ${stringOrNone(protocol)})`;
+    return code;
+  }
+
+  sendWebsocketMessageFunctionFactory(options: {
+    instanceId: string;
+    kernelId: string;
+    wsUrl: string;
+    message: string;
+  }): string {
+    const { instanceId, kernelId, wsUrl, message } = options;
+    const code = `await ${this._SEND_WEBSOCKET_FUNCTION}("${instanceId}", "${kernelId}", "${wsUrl}", '''${message}''')`;
+    return code;
+  }
+
+  async disposePythonServer(): Promise<void> {
+    await this.executeCode({
+      code: '__jupyterpack_streamlit_dispose()'
+    });
+  }
+
+  private _GET_RESPONSE_FUNCTION = '__jupyterpack_streamlit_get_response';
+  private _OPEN_WEBSOCKET_FUNCTION = '__jupyterpack_streamlit_open_ws';
+  private _SEND_WEBSOCKET_FUNCTION =
+    '__jupyterpack_streamlit_receive_ws_message';
+}