@basthon/kernel-base 0.62.21 → 0.63.0

package/lib/worker/kernel-main.js

@@ -0,0 +1,203 @@
+import { wrap, createEndpoint, transfer } from "comlink";
+import { NotImplementedError } from "../commons";
+import { KernelBase } from "../kernel";
+// mock Worker to support sandboxing and keep webpack compatibility
+if (globalThis.Worker.name !== "SandboxedWorker") {
+    class SandboxedWorker extends Worker {
+        constructor(url, options) {
+            if (options?.sandboxed === true) {
+                const absURL = new URL(url).href;
+                const assetsURL = absURL.substring(0, absURL.lastIndexOf("/"));
+                // worker created with a 'data' url scheme has opaque origin
+                // https://html.spec.whatwg.org/multipage/workers.html#script-settings-for-workers:concept-origin-opaque
+                const workerScript = `self._assetsURL = "${assetsURL}";
+           globalThis = {importScripts: self.importScripts, location: "${url}"};
+           importScripts("${url}");
+           globalThis = self;`.replace(/\s/g, "");
+                super(`data:text/javascript;base64,${btoa(workerScript)}`);
+            }
+            else {
+                super(url, options);
+            }
+        }
+    }
+    globalThis.Worker = SandboxedWorker;
+}
+export class KernelMainBase extends KernelBase {
+    constructor(options) {
+        super(options);
+        this._legacy = false;
+        this._legacy = options?.legacy === true;
+    }
+    /**
+     * Is this a legacy worker? i.e. not runing in a worker.
+     */
+    isLegacy() {
+        return this._legacy;
+    }
+    /**
+     * Is this kernel safe? (loaded in an isolated web worker)
+     */
+    safeKernel() {
+        return !this.isLegacy();
+    }
+    /**
+     * Get a new worker (should be overloaded).
+     */
+    newWorker() {
+        throw new NotImplementedError("newWorker");
+    }
+    /**
+     * Import the code for a legacy worker (running in the main thread).
+     */
+    async importLegacyWorker() {
+        throw new NotImplementedError("importLegacyWorker");
+    }
+    /**
+     * Create the kernel webworker and setup communication.
+     */
+    async setupWorker() {
+        const comlinkChannel = new MessageChannel();
+        if (this.isLegacy()) {
+            // running in main thread via message channel (fake worker)
+            //@ts-ignore
+            self._commPort = comlinkChannel.port1;
+            await this.importLegacyWorker();
+        }
+        else {
+            // setup the kernel worker
+            const worker = this.newWorker();
+            // send communication port
+            worker.postMessage(comlinkChannel.port1, [comlinkChannel.port1]);
+            this._worker = worker;
+        }
+        const KernelWorker = wrap(comlinkChannel.port2);
+        // why typescript complains about this call? yet, this is advertised in
+        // https://github.com/GoogleChromeLabs/comlink/blob/dffe9050f63b1b39f30213adeb1dd4b9ed7d2594/docs/examples/03-classes-example/index.html#L15
+        //@ts-ignore
+        this._remote = await new KernelWorker(this._options);
+        // setup the communication channel
+        const commChannel = new MessageChannel();
+        commChannel.port2.onmessage = (e) => this.processWorkerMessage(e.data);
+        await this.remote?.setCommPort(transfer(commChannel.port1, [commChannel.port1]));
+        // WARNING: without the setCommPort call, KernelWorker (proxy)
+        // is garbage collected and comlink releases the proxy,
+        // breaking all the communications... (don't know really why)
+        // If you experience this, just keep a ref on the proxy with e.g.
+        // this.PROXY = KernelWorker
+    }
+    /**
+     * Start the worker.
+     */
+    async _start() {
+        await super._start();
+        // setup the proxy worker
+        const proxyWorker = new Worker(new URL("./comlink-proxy.js", import.meta.url), {
+            type: "module",
+        });
+        const proxy = wrap(proxyWorker);
+        // setup the kernel worker
+        await this.setupWorker();
+        // bind ports
+        const port = await proxy[createEndpoint]();
+        await this.remote?.setProxyPort(transfer(port, [port]));
+        // init remote
+        await this.remote?.init();
+    }
+    /**
+     * Comlink remote getter.
+     */
+    get remote() {
+        return this._remote;
+    }
+    /**
+     * Stop the worker.
+     */
+    async _stop() {
+        this._worker?.terminate();
+        this._worker = undefined;
+        this._remote = undefined;
+        await super._stop();
+    }
+    /**
+     * Process messages from worker (if any).
+     */
+    processWorkerMessage(msg) {
+        switch (msg.type) {
+            case "stream":
+                const { stream, text } = msg.content;
+                const dataEvent = this.clone(msg.data);
+                dataEvent.stream = stream;
+                dataEvent.content = text;
+                this.dispatchEvent("eval.output", dataEvent);
+                break;
+            case "download":
+                const { content, filename } = msg.content;
+                this.download(content, filename);
+                break;
+            case "display":
+                this.dispatchEvent("eval.display", msg.data);
+                break;
+            case "input":
+                const { prompt, password, data, sab } = msg.content;
+                (async () => {
+                    const res = (await this.inputAsync(prompt, password, data));
+                    const encodedRes = new TextEncoder().encode(res);
+                    new Uint8Array(sab).set(encodedRes.subarray(0, sab.byteLength));
+                    Atomics.notify(new Int32Array(sab), 0, 1);
+                })();
+                break;
+            default: // ignoring (probably a comlink message)
+        }
+    }
+    async evalAsync(code, outCallback, errCallback, data = null) {
+        // force interactivity in all modes
+        data.interactive = true;
+        this._execution_count++;
+        // evaluation
+        const execResult = (await this.remote?.eval(data, code));
+        // return result
+        let result;
+        switch (execResult?.status) {
+            case "ok":
+                result = execResult.result;
+                break;
+            case "error":
+                break;
+        }
+        return [result, this._execution_count];
+    }
+    /**
+     * Put a file on the local (emulated) filesystem.
+     */
+    async putFile(filename, content) {
+        await super.putFile(filename, content);
+        await this.remote?.putFile(filename, content);
+    }
+    /**
+     * Put an importable module on the local (emulated) filesystem
+     * and load dependencies.
+     */
+    async putModule(filename, content) {
+        await super.putModule(filename, content);
+        await this.remote?.putModule(filename, content);
+    }
+    /**
+     * Complete a code at the end (usefull for tab completion).
+     *
+     * Returns an array of two elements: the list of completions
+     * and the start index.
+     */
+    async complete(code) {
+        return (await this.remote?.complete(code)) ?? [];
+    }
+    /**
+     * Tell wether we should wait for more code or if it can
+     * be run as is.
+     *
+     * Useful to set ps1/ps2 in console prompt.
+     */
+    async more(code) {
+        return (await this.remote?.more(code)) ?? false;
+    }
+}

package/lib/worker/kernel-worker.d.ts

@@ -0,0 +1,120 @@
+import { Remote } from "comlink";
+import { ProxyFetch } from "./proxy";
+declare global {
+    interface DedicatedWorkerGlobalScope {
+        _remoteProxy: Remote<ProxyFetch>;
+        basthon: KernelWorkerBase;
+        _prompt: (p?: string, _d?: string) => string | null;
+    }
+}
+type ExecuteResult = {
+    data: any;
+    status: "ok" | "error";
+    result?: any;
+    error?: any;
+};
+declare class KernelWorkerBase {
+    protected _options: any;
+    protected _legacy: boolean;
+    private _ready;
+    protected __eval_data__: any;
+    protected postMessage?: InstanceType<typeof MessagePort>["postMessage"];
+    constructor(options?: any);
+    /**
+     * Is this a legacy worker? i.e. not runing in a worker.
+     */
+    isLegacy(): boolean;
+    protected _init(): Promise<void>;
+    /**
+     * Initialize the kernel. It is called by initWorker in KernelBase.
+     */
+    init(): Promise<void>;
+    /**
+     * Wait for the kernel to be ready.
+     */
+    ready(): Promise<void>;
+    /**
+     * Set communication port.
+     */
+    setCommPort(port: MessagePort): void;
+    /**
+     * Set the comlink communication port to proxy fetch.
+     */
+    setProxyPort(port: MessagePort): void;
+    /**
+     * Send stream to stdout/stderr.
+     */
+    private sendStream;
+    /**
+     * Send stream to stdout.
+     */
+    protected sendStdoutStream(data: any, text: string): void;
+    /**
+     * Send stream to stderr.
+     */
+    protected sendStderrStream(data: any, text: string): void;
+    protected blobToDataURL(blob: Blob): Promise<string>;
+    /**
+     * Display a blob (only png images are supported).
+     */
+    protected displayBlob(blob: Blob, data: any): Promise<void>;
+    /**
+     * Display an element like a canvas, ... in the frontend.
+     */
+    display(element: any): void;
+    /**
+     * Evaluate code and return an object with mime type keys and string value.
+     * To be overloaded.
+     */
+    protected _eval(data: any, code: string): Promise<undefined | {
+        [key: string]: string;
+    }>;
+    /**
+     * Evaluate a string of code. To be called by main thread (through comlink).
+     */
+    eval(data: any, code: string): Promise<ExecuteResult>;
+    /**
+     * Put a file on the local (emulated) filesystem.
+     */
+    putFile(filename: string, content: ArrayBuffer): void;
+    /**
+     * Put an importable module on the local (emulated) filesystem
+     * and load dependencies.
+     */
+    putModule(filename: string, content: ArrayBuffer): void;
+    /**
+     * Complete the submited code.
+     */
+    complete(code: string): Promise<[string[], number] | []>;
+    /**
+     * Tell wether we should wait for more code or if it can
+     * be run as is.
+     *
+     * Useful to set ps1/ps2 in console prompt.
+     */
+    more(code: string): Promise<boolean>;
+    /**
+     * Send a message to the main thread to download a file.
+     */
+    download(content: Uint8Array | string, filename: string): void;
+    /**
+     * Use JSON stringify/parse to copy an object.
+     */
+    protected clone(data: any): any;
+    /**
+     * Sync sleep for a certain duration in seconds.
+     */
+    sleep(duration: number): void;
+    /**
+     * Decode the string contained in a SharedArrayBuffer to
+     * bypas this issue:
+     * https://github.com/whatwg/encoding/issues/172
+     */
+    private decodeSharedArrayBuffer;
+    /**
+     * Sync input that does not use the ugly prompt but send
+     * an eval.input event tho the main thread and wait for the response.
+     */
+    input(prompt: string | null | undefined, password?: boolean): string;
+}
+export { ExecuteResult, KernelWorkerBase };

package/lib/worker/kernel-worker.js

@@ -0,0 +1,258 @@
+import { PromiseDelegate } from "promise-delegate";
+import { NotImplementedError, mockFetch } from "../commons";
+import { wrap } from "comlink";
+class KernelWorkerBase {
+    constructor(options) {
+        this._ready = new PromiseDelegate();
+        this._options = options;
+        this._legacy = options?.legacy === true;
+        /* we mock the fetch function to:
+         *  - redirect queries to local FS
+         *  - bypass the worker's opaque origin
+         */
+        mockFetch(this.isLegacy());
+        // useful for ffi
+        self.basthon = this;
+        // global prompt redirected to our input
+        if (self._prompt == null)
+            self._prompt = globalThis.prompt;
+        globalThis.prompt = (prompt) => this.input(prompt);
+    }
+    /**
+     * Is this a legacy worker? i.e. not runing in a worker.
+     */
+    isLegacy() {
+        return this._legacy;
+    }
+    /*
+     * Initialize the kernel. Should be overloaded.
+     */
+    async _init() { }
+    /**
+     * Initialize the kernel. It is called by initWorker in KernelBase.
+     */
+    async init() {
+        try {
+            await this._init();
+            this._ready.resolve();
+        }
+        catch (e) {
+            this._ready.reject(e);
+        }
+    }
+    /**
+     * Wait for the kernel to be ready.
+     */
+    ready() {
+        return this._ready.promise;
+    }
+    /**
+     * Set communication port.
+     */
+    setCommPort(port) {
+        this.postMessage = port.postMessage.bind(port);
+    }
+    /**
+     * Set the comlink communication port to proxy fetch.
+     */
+    setProxyPort(port) {
+        self._remoteProxy = wrap(port);
+    }
+    /**
+     * Send stream to stdout/stderr.
+     */
+    sendStream(data, stream, text) {
+        this.postMessage?.({
+            data,
+            type: "stream",
+            content: { stream, text },
+        });
+    }
+    /**
+     * Send stream to stdout.
+     */
+    sendStdoutStream(data, text) {
+        this.sendStream(data, "stdout", text);
+    }
+    /**
+     * Send stream to stderr.
+     */
+    sendStderrStream(data, text) {
+        this.sendStream(data, "stderr", text);
+    }
+    async blobToDataURL(blob) {
+        return await new Promise((resolve) => {
+            const reader = new FileReader();
+            reader.onloadend = () => resolve(reader.result);
+            reader.readAsDataURL(blob);
+        });
+    }
+    /**
+     * Display a blob (only png images are supported).
+     */
+    async displayBlob(blob, data) {
+        if (data == null)
+            data = this.clone(this.__eval_data__);
+        const dataURL = await this.blobToDataURL(blob);
+        const pngPrefix = "data:image/png;base64,";
+        if (dataURL.startsWith(pngPrefix)) {
+            const png = dataURL.slice(pngPrefix.length);
+            data["display_type"] = "multiple";
+            data["content"] = { "image/png": png };
+        }
+        else {
+            return;
+        }
+        this.postMessage?.({ data, type: "display" });
+    }
+    /**
+     * Display an element like a canvas, ... in the frontend.
+     */
+    display(element) {
+        const data = this.clone(this.__eval_data__);
+        if (globalThis.OffscreenCanvas != null &&
+            element instanceof OffscreenCanvas) {
+            element
+                .convertToBlob()
+                .then((blob) => this.displayBlob(blob, data));
+        }
+        else {
+            data["display_type"] = "multiple";
+            data["content"] = element;
+            this.postMessage?.({ data, type: "display" });
+        }
+    }
+    /**
+     * Evaluate code and return an object with mime type keys and string value.
+     * To be overloaded.
+     */
+    async _eval(data, code) {
+        throw new NotImplementedError("_eval");
+    }
+    /**
+     * Evaluate a string of code. To be called by main thread (through comlink).
+     */
+    async eval(data, code) {
+        this.__eval_data__ = data;
+        try {
+            const result = await this._eval(data, code);
+            return { data, status: "ok", result };
+        }
+        catch (e) {
+            const { name, stack, message } = e;
+            this.sendStderrStream(data, e.toString());
+            return {
+                data,
+                status: "error",
+                error: {
+                    name: name,
+                    value: message,
+                    traceback: stack?.toString(),
+                },
+            };
+        }
+    }
+    /**
+     * Put a file on the local (emulated) filesystem.
+     */
+    putFile(filename, content) {
+        throw new NotImplementedError("putFile");
+    }
+    /**
+     * Put an importable module on the local (emulated) filesystem
+     * and load dependencies.
+     */
+    putModule(filename, content) {
+        throw new NotImplementedError("putModule");
+    }
+    /**
+     * Complete the submited code.
+     */
+    async complete(code) {
+        return [];
+    }
+    /**
+     * Tell wether we should wait for more code or if it can
+     * be run as is.
+     *
+     * Useful to set ps1/ps2 in console prompt.
+     */
+    async more(code) {
+        throw new NotImplementedError("more");
+    }
+    /**
+     * Send a message to the main thread to download a file.
+     */
+    download(content, filename) {
+        this.postMessage?.({
+            type: "download",
+            content: { content, filename },
+        });
+    }
+    /**
+     * Use JSON stringify/parse to copy an object.
+     */
+    clone(data) {
+        return JSON.parse(JSON.stringify(data));
+    }
+    /**
+     * Sync sleep for a certain duration in seconds.
+     */
+    sleep(duration) {
+        duration = duration * 1000; // to milliseconds
+        if (globalThis.crossOriginIsolated && !this.isLegacy()) {
+            const sab = new Int32Array(new SharedArrayBuffer(4));
+            Atomics.wait(sab, 0, 0, duration);
+            return;
+        }
+        else {
+            const t0 = Date.now();
+            // active sleep
+            while (Date.now() - t0 < duration)
+                ;
+        }
+    }
+    /**
+     * Decode the string contained in a SharedArrayBuffer to
+     * bypas this issue:
+     * https://github.com/whatwg/encoding/issues/172
+     */
+    decodeSharedArrayBuffer(sab) {
+        const array = new Uint8Array(sab);
+        let length = array.length;
+        for (; length > 0 && array[length - 1] === 0; length--)
+            ;
+        if (length === 0)
+            return "";
+        const cropped = new Uint8Array(length);
+        cropped.set(array.subarray(0, length));
+        try {
+            return new TextDecoder().decode(cropped);
+        }
+        catch (e) {
+            if (array.length === length)
+                throw new Error("String too long, can't decode");
+            throw e;
+        }
+    }
+    /**
+     * Sync input that does not use the ugly prompt but send
+     * an eval.input event tho the main thread and wait for the response.
+     */
+    input(prompt, password = false) {
+        if (globalThis.crossOriginIsolated && !this.isLegacy()) {
+            const data = this.clone(this.__eval_data__);
+            const sab = new SharedArrayBuffer(1024);
+            this.postMessage?.({
+                type: "input",
+                content: { data, prompt, password, sab },
+            });
+            Atomics.wait(new Int32Array(sab), 0, 0);
+            return this.decodeSharedArrayBuffer(sab);
+        }
+        else {
+            return self._prompt(prompt ?? "") ?? "";
+        }
+    }
+}
+export { KernelWorkerBase };

package/lib/worker/proxy.d.ts

@@ -0,0 +1,10 @@
+/**
+ * A class for a proxy that allows a worker with opaque origin
+ * to fetch assets.
+ */
+export declare class ProxyFetch {
+    fetch(url: string, init: RequestInit | undefined): Promise<{
+        body: any;
+        options: any | undefined;
+    }>;
+}

package/lib/worker/proxy.js

@@ -0,0 +1,30 @@
+import { transfer } from "comlink";
+/**
+ * A class for a proxy that allows a worker with opaque origin
+ * to fetch assets.
+ */
+export class ProxyFetch {
+    async fetch(url, init) {
+        const parentURL = (s) => s.substring(0, s.lastIndexOf("/"));
+        const assetsURL = parentURL(new URL(globalThis.location.toString()).href);
+        const baseURL = parentURL(new URL(url).href);
+        if (!baseURL.startsWith(assetsURL))
+            throw new Error(`URL ${url} is outside of proxy' scope`);
+        // force credentials omition
+        if (init?.credentials !== "omit")
+            throw new Error("Request tries to use credentials");
+        const response = await globalThis.fetch(url, init);
+        const buffer = await response.arrayBuffer();
+        return {
+            body: transfer(buffer, [buffer]),
+            options: {
+                status: response.status,
+                statusText: response.statusText,
+                // wrong types for headers in worker...
+                // see https://github.com/microsoft/TypeScript/pull/15050/files#diff-1fe269c5fa7bcd3a57e9e6a91ce3566d3e3a5c38b21e6ff0d78ad0dd090f8d3aR24
+                // @ts-ignore
+                headers: new Map(response.headers), // Headers are not clonable
+            },
+        };
+    }
+}

package/lib/worker/worker-utils.d.ts

@@ -0,0 +1,6 @@
+declare global {
+    interface DedicatedWorkerGlobalScope {
+        _commPort?: MessagePort;
+    }
+}
+export declare const expose: (kwc: any) => void;

package/lib/worker/worker-utils.js

@@ -0,0 +1,21 @@
+import { expose as exposeComlink } from "comlink";
+const insideWorker = () => 
+//@ts-ignore
+typeof WorkerGlobalScope !== "undefined" && self instanceof WorkerGlobalScope;
+export const expose = (kwc) => {
+    if (insideWorker()) {
+        // worker
+        const listener = (msg) => {
+            if (!(msg?.data instanceof MessagePort))
+                return;
+            removeEventListener("message", listener);
+            exposeComlink(kwc, msg.data);
+        };
+        addEventListener("message", listener);
+    }
+    else {
+        // main thread
+        exposeComlink(kwc, self._commPort);
+        self._commPort = undefined; // no more needed
+    }
+};