@pyscript/core 0.0.3 → 0.0.5

package/README.md

@@ -4,9 +4,13 @@
 
 ---
 
+## Documentation
+
+Please read [the documentation page](./docs/README.md) to know all the user-facing details around this module.
+
 ## Development
 
-The working folder (source code of truth) is the `./esm` one, while the `./cjs` is populated as dual module and to test (but it's 1:1 code, no trnaspilation except for imports/exports).
+The working folder (source code of truth) is the `./esm` one, while the `./cjs` is populated as dual module and to test (but it's 1:1 code, no transpilation except for imports/exports).
 
 ```sh
 # install all dependencies needed by core
@@ -19,7 +23,7 @@ This project requires some automatic artifact creation to:
 
     * create a _Worker_ as a _Blob_ based on the same code used by this repo
     * create automatically the list of runtimes available via the module
-    * create the `core.js` file used by most integration tests
+    * create the `core.js` or the `pyscript.js` file used by most integration tests
     * create a sha256 version of the Blob content for CSP cases
 
 Accordingly, to build latest project:
@@ -31,3 +35,11 @@ npm run build
 # optionally spin a server with CORS, COOP, and COEP enabled
 npm run server
 ```
+
+If **no minification** is desired or helpful while debugging potential issues, please use `NO_MIN=1` in front of the _build_ step:
+
+```sh
+NO_MIN=1 npm run build
+
+npm run server
+```

package/cjs/custom/pyscript.js

@@ -91,14 +91,10 @@ document.head.appendChild(document.createElement("style")).textContent = `
         env: "py-script",
         interpreter: "pyodide",
         codeBeforeRunWorker() {
-            const { codeBeforeRunWorker: set } = hooks;
-            const prefix = 'print("codeBeforeRunWorker")';
-            return [prefix].concat(...set).join("\n");
+            return [...hooks.codeBeforeRunWorker].join("\n");
         },
         codeAfterRunWorker() {
-            const { codeAfterRunWorker: set } = hooks;
-            const prefix = 'print("codeAfterRunWorker")';
-            return [prefix].concat(...set).join("\n");
+            return [...hooks.codeAfterRunWorker].join("\n");
         },
         onBeforeRun(pyodide, element) {
             bootstrapNodeAndPlugins(pyodide, element, before, "onBeforeRun");
@@ -119,10 +115,10 @@ document.head.appendChild(document.createElement("style")).textContent = `
         onAfterRunAsync(pyodide, element) {
             bootstrapNodeAndPlugins(pyodide, element, after, "onAfterRunAsync");
         },
-        async onRuntimeReady(pyodide, element) {
+        async onInterpreterReady(pyodide, element) {
             // allows plugins to do whatever they want with the element
             // before regular stuff happens in here
-            for (const callback of hooks.onRuntimeReady)
+            for (const callback of hooks.onInterpreterReady)
                 callback(pyodide, element);
             if (isScript(element)) {
                 const {
@@ -183,7 +179,7 @@ const hooks = {
     /** @type {Set<function>} */
     onAfterRunAsync: new Set(),
     /** @type {Set<function>} */
-    onRuntimeReady: new Set(),
+    onInterpreterReady: new Set(),
 
     /** @type {Set<string>} */
     codeBeforeRunWorker: new Set(),

package/cjs/custom-types.js

@@ -126,8 +126,6 @@ const handleCustomType = (node) => {
                             afterRunAsync: codeAfterRunWorkerAsync,
                         });
 
-                        module.setGlobal(interpreter, "XWorker", XWorker);
-
                         const resolved = {
                             type,
                             interpreter,

package/cjs/custom.js

@@ -48,7 +48,7 @@ const handleCustomType = (node) => {
                     version,
                     config,
                     env,
-                    onRuntimeReady,
+                    onInterpreterReady,
                 } = options;
                 const name = getRuntimeID(runtime, version);
                 const id = env || `${name}${config ? `|${config}` : ""}`;
@@ -69,7 +69,7 @@ const handleCustomType = (node) => {
                         onAfterRunAsync,
                     } = options;
 
-                    const hooks = new Hook(options);
+                    const hooks = new Hook(interpreter, options);
 
                     const XWorker = function XWorker(...args) {
                         return Worker.apply(hooks, args);
@@ -123,7 +123,7 @@ const handleCustomType = (node) => {
 
                     resolve(resolved);
 
-                    onRuntimeReady?.(resolved, node);
+                    onInterpreterReady?.(resolved, node);
                 });
             }
         }
@@ -137,17 +137,17 @@ exports.handleCustomType = handleCustomType;
 const registry = new Map();
 
 /**
- * @typedef {Object} PluginOptions custom configuration
+ * @typedef {Object} CustomOptions custom configuration
  * @prop {'pyodide' | 'micropython' | 'wasmoon' | 'ruby-wasm-wasi'} interpreter the interpreter to use
  * @prop {string} [version] the optional interpreter version to use
  * @prop {string} [config] the optional config to use within such interpreter
- * @prop {(environment: object, node: Element) => void} [onRuntimeReady] the callback that will be invoked once
+ * @prop {(environment: object, node: Element) => void} [onInterpreterReady] the callback that will be invoked once
  */
 
 /**
  * Allows custom types and components on the page to receive interpreters to execute any code
  * @param {string} type the unique `<script type="...">` identifier
- * @param {PluginOptions} options the custom type configuration
+ * @param {CustomOptions} options the custom type configuration
  */
 const define = (type, options) => {
     if (defaultRegistry.has(type) || registry.has(type))

package/cjs/interpreter/_python.js

@@ -10,12 +10,14 @@ const runAsync = (interpreter, code) =>
     interpreter.runPythonAsync(clean(code));
 exports.runAsync = runAsync;
 
-const setGlobal = (interpreter, name, value) =>
+const setGlobal = (interpreter, name, value) => {
     interpreter.globals.set(name, value);
+};
 exports.setGlobal = setGlobal;
 
-const deleteGlobal = (interpreter, name) =>
+const deleteGlobal = (interpreter, name) => {
     interpreter.globals.delete(name);
+};
 exports.deleteGlobal = deleteGlobal;
 
 const writeFile = ({ FS }, path, buffer) =>

package/cjs/interpreter/micropython.js

@@ -1,12 +1,6 @@
 'use strict';
 const { fetchPaths, stdio } = require("./_utils.js");
-const {
-    run,
-    runAsync,
-    setGlobal,
-    deleteGlobal,
-    writeFile
-} = require("./_python.js");
+const { run, setGlobal, deleteGlobal, writeFile } = require("./_python.js");
 
 const type = "micropython";
 
@@ -14,19 +8,23 @@ const type = "micropython";
 /* c8 ignore start */
 module.exports = {
     type,
-    module: (version = "1.20.0-253") =>
+    module: (version = "1.20.0-268") =>
         `https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript@${version}/micropython.mjs`,
     async engine({ loadMicroPython }, config, url) {
         const { stderr, stdout, get } = stdio();
         url = url.replace(/\.m?js$/, ".wasm");
-        const runtime = await get(loadMicroPython({ stderr, stdout, url }));
-        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
-        return runtime;
+        const interpreter = await get(loadMicroPython({ stderr, stdout, url }));
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
     },
     setGlobal,
     deleteGlobal,
     run,
-    runAsync,
+    // TODO: MicroPython doesn't have a Pyodide like top-level await,
+    //       this method should still not throw errors once invoked
+    async runAsync(...args) {
+        return this.run(...args);
+    },
     writeFile,
 };
 /* c8 ignore stop */

package/cjs/interpreter/webr.js

@@ -0,0 +1,48 @@
+'use strict';
+const { fetchPaths, stdio, writeFile } = require("./_utils.js");
+
+const type = "webr";
+
+const io = new WeakMap;
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type,
+    module: (version = "0.1.1") =>
+        `https://webr.r-wasm.org/v${version}/webr.mjs`,
+    async engine({ WebR }, config) {
+        const { stderr, stdout, get } = stdio();
+        const webR = new WebR();
+        await webR.init();
+        const interpreter = await get(new webR.Shelter());
+        io.set(interpreter, { webR, stderr, stdout });
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
+    },
+    setGlobal() {
+        // UNSUPPORTED
+        // const { webR } = io.get(interpreter);
+        // return webR.objs.globalEnv.bind(name, value);
+    },
+    deleteGlobal() {
+        // UNSUPPORTED
+        // const { webR } = io.get(interpreter);
+        // return webR.objs.globalEnv.bind(name, void 0);
+    },
+    run(interpreter, code) {
+        return this.runAsync(interpreter, code);
+    },
+    async runAsync(interpreter, code) {
+        const ioHandler = io.get(interpreter);
+        const { output, result } = await interpreter.captureR(code);
+        for (const { type, data } of output)
+            ioHandler[type](data);
+        return result;
+    },
+    writeFile: (interpreter, path, buffer) => {
+        const { webR } = io.get(interpreter);
+        return writeFile(webR.FS, path, buffer);
+    },
+};
+/* c8 ignore stop */

package/cjs/plugins.js

@@ -0,0 +1,89 @@
+'use strict';
+const { $$ } = require("basic-devtools");
+
+const { getDetails } = require("./script-handler.js");
+const { registry, configs } = require("./runtimes.js");
+const { getRuntimeID } = require("./loader.js");
+const { io } = require("./runtime/_utils.js");
+
+const PLUGINS_SELECTORS = [];
+exports.PLUGINS_SELECTORS = PLUGINS_SELECTORS;
+
+/**
+ * @typedef {Object} Runtime plugin configuration
+ * @prop {string} type the runtime type
+ * @prop {object} runtime the bootstrapped runtime
+ * @prop {(url:string, options?: object) => Worker} XWorker an XWorker constructor that defaults to same runtime on the Worker.
+ * @prop {object} config a cloned config used to bootstrap the runtime
+ * @prop {(code:string) => any} run an utility to run code within the runtime
+ * @prop {(code:string) => Promise<any>} runAsync an utility to run code asynchronously within the runtime
+ * @prop {(path:string, data:ArrayBuffer) => void} writeFile an utility to write a file in the virtual FS, if available
+ */
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+/**
+ * @param {Element} node any DOM element registered via plugin.
+ */
+const handlePlugin = (node) => {
+    for (const name of PLUGINS_SELECTORS) {
+        if (node.matches(name)) {
+            const { options, known } = plugins.get(name);
+            if (!known.has(node)) {
+                known.add(node);
+                const { type, version, config, env, onRuntimeReady } = options;
+                const name = getRuntimeID(type, version);
+                const id = env || `${name}${config ? `|${config}` : ""}`;
+                const { runtime: engine, XWorker } = getDetails(
+                    type,
+                    id,
+                    name,
+                    version,
+                    config,
+                );
+                engine.then((runtime) => {
+                    const module = registry.get(type);
+                    onRuntimeReady(node, {
+                        type,
+                        runtime,
+                        XWorker,
+                        io: io.get(runtime),
+                        config: structuredClone(configs.get(name)),
+                        run: module.run.bind(module, runtime),
+                        runAsync: module.runAsync.bind(module, runtime),
+                    });
+                });
+            }
+        }
+    }
+};
+exports.handlePlugin = handlePlugin;
+
+/**
+ * @type {Map<string, {options:object, known:WeakSet<Element>}>}
+ */
+const plugins = new Map();
+
+/**
+ * @typedef {Object} PluginOptions plugin configuration
+ * @prop {string} type the runtime/interpreter type to receive
+ * @prop {string} [version] the optional runtime version to use
+ * @prop {string} [config] the optional config to use within such runtime
+ * @prop {string} [env] the optional environment to use
+ * @prop {(node: Element, runtime: Runtime) => void} onRuntimeReady the callback that will be invoked once
+ */
+
+/**
+ * Allows plugins and components on the page to receive runtimes to execute any code.
+ * @param {string} name the unique plugin name
+ * @param {PluginOptions} options the plugin configuration
+ */
+const registerPlugin = (name, options) => {
+    if (PLUGINS_SELECTORS.includes(name))
+        throw new Error(`plugin ${name} already registered`);
+    PLUGINS_SELECTORS.push(name);
+    plugins.set(name, { options, known: new WeakSet() });
+    $$(name).forEach(handlePlugin);
+};
+exports.registerPlugin = registerPlugin;
+/* c8 ignore stop */

package/cjs/runtime/_python.js

@@ -0,0 +1,34 @@
+'use strict';
+const { clean, writeFile: writeFileUtil } = require("./_utils.js");
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+const run = (runtime, code) => runtime.runPython(clean(code));
+exports.run = run;
+
+const runAsync = (runtime, code) => runtime.runPythonAsync(clean(code));
+exports.runAsync = runAsync;
+
+function runEvent(runtime, code, key) {
+    code = `import js;event=js.__events.get(${key});${code}`;
+    return this.run(runtime, code);
+}
+exports.runEvent = runEvent
+
+const worker = (method) =>
+    function (runtime, code, xworker) {
+        code = `from js import xworker;${code}`;
+        globalThis.xworker = xworker;
+        return this[method](runtime, code);
+    };
+
+const runWorker = worker("run");
+exports.runWorker = runWorker;
+
+const runWorkerAsync = worker("runAsync");
+exports.runWorkerAsync = runWorkerAsync;
+
+const writeFile = ({ FS }, path, buffer) =>
+    writeFileUtil(FS, path, buffer);
+exports.writeFile = writeFile;
+/* c8 ignore stop */

package/cjs/runtime/_utils.js

@@ -0,0 +1,141 @@
+'use strict';
+require("@ungap/with-resolvers");
+
+const { getBuffer } = require("../fetch-utils.js");
+const { absoluteURL } = require("../utils.js");
+
+/**
+ * Trim code only if it's a single line that prettier or other tools might have modified.
+ * @param {string} code code that might be a single line
+ * @returns {string}
+ */
+const clean = (code) =>
+    code.replace(/^[^\r\n]+$/, (line) => line.trim());
+exports.clean = clean;
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+const io = new WeakMap();
+exports.io = io;
+const stdio = (init) => {
+    const context = init || console;
+    const localIO = {
+        stderr: (context.stderr || console.error).bind(context),
+        stdout: (context.stdout || console.log).bind(context),
+    };
+    return {
+        stderr: (...args) => localIO.stderr(...args),
+        stdout: (...args) => localIO.stdout(...args),
+        async get(engine) {
+            const runtime = await engine;
+            io.set(runtime, localIO);
+            return runtime;
+        },
+    };
+};
+exports.stdio = stdio;
+/* c8 ignore stop */
+
+// This should be the only helper needed for all Emscripten based FS exports
+const writeFile = (FS, path, buffer) => {
+    const { parentPath, name } = FS.analyzePath(path, true);
+    FS.mkdirTree(parentPath);
+    return FS.writeFile([parentPath, name].join("/"), new Uint8Array(buffer), {
+        canOwn: true,
+    });
+};
+exports.writeFile = writeFile;
+
+// This is instead a fallback for Lua or others
+const writeFileShim = (FS, path, buffer) => {
+    path = resolve(FS, path);
+    mkdirTree(FS, dirname(path));
+    return FS.writeFile(path, new Uint8Array(buffer), { canOwn: true });
+};
+exports.writeFileShim = writeFileShim;
+
+const dirname = (path) => {
+    const tree = path.split("/");
+    tree.pop();
+    return tree.join("/");
+};
+
+const mkdirTree = (FS, path) => {
+    const current = [];
+    for (const branch of path.split("/")) {
+        current.push(branch);
+        if (branch) FS.mkdir(current.join("/"));
+    }
+};
+
+const resolve = (FS, path) => {
+    const tree = [];
+    for (const branch of path.split("/")) {
+        switch (branch) {
+            case "":
+                break;
+            case ".":
+                break;
+            case "..":
+                tree.pop();
+                break;
+            default:
+                tree.push(branch);
+        }
+    }
+    return [FS.cwd()].concat(tree).join("/").replace(/^\/+/, "/");
+};
+
+const { all, isArray } = require("../utils.js");
+
+const calculateFetchPaths = (config_fetch) => {
+    // REQUIRES INTEGRATION TEST
+    /* c8 ignore start */
+    for (const { files, to_file, from = "" } of config_fetch) {
+        if (files !== undefined && to_file !== undefined)
+            throw new Error(
+                `Cannot use 'to_file' and 'files' parameters together!`,
+            );
+        if (files === undefined && to_file === undefined && from.endsWith("/"))
+            throw new Error(
+                `Couldn't determine the filename from the path ${from}, please supply 'to_file' parameter.`,
+            );
+    }
+    /* c8 ignore stop */
+    return config_fetch.flatMap(
+        ({ from = "", to_folder = ".", to_file, files }) => {
+            if (isArray(files))
+                return files.map((file) => ({
+                    url: joinPaths([from, file]),
+                    path: joinPaths([to_folder, file]),
+                }));
+            const filename = to_file || from.slice(1 + from.lastIndexOf("/"));
+            return [{ url: from, path: joinPaths([to_folder, filename]) }];
+        },
+    );
+};
+
+const joinPaths = (parts) => {
+    const res = parts
+        .map((part) => part.trim().replace(/(^[/]*|[/]*$)/g, ""))
+        .filter((p) => p !== "" && p !== ".")
+        .join("/");
+
+    return parts[0].startsWith("/") ? `/${res}` : res;
+};
+
+const fetchResolved = (config_fetch, url) =>
+    fetch(absoluteURL(url, base.get(config_fetch)));
+
+const base = new WeakMap();
+exports.base = base;
+
+const fetchPaths = (module, runtime, config_fetch) =>
+    all(
+        calculateFetchPaths(config_fetch).map(({ url, path }) =>
+            fetchResolved(config_fetch, url)
+                .then(getBuffer)
+                .then((buffer) => module.writeFile(runtime, path, buffer)),
+        ),
+    );
+exports.fetchPaths = fetchPaths;

package/cjs/runtime/micropython.js

@@ -0,0 +1,40 @@
+'use strict';
+const { fetchPaths, stdio } = require("./_utils.js");
+const {
+    run,
+    runAsync,
+    runEvent,
+    runWorker,
+    runWorkerAsync,
+    writeFile
+} = require("./_python.js");
+
+const type = "micropython";
+
+let patchPromise = true;
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type: [type, "mpy"],
+    module: () => `http://localhost:8080/micropython/micropython.mjs`,
+    async engine({ loadMicroPython }, config, url) {
+        // @bug https://github.com/micropython/micropython/issues/11749
+        if (patchPromise) {
+            patchPromise = false;
+            globalThis.Promise = class extends Promise {};
+        }
+        const { stderr, stdout, get } = stdio();
+        url = url.replace(/\.m?js$/, ".wasm");
+        const runtime = await get(loadMicroPython({ stderr, stdout, url }));
+        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
+        return runtime;
+    },
+    run,
+    runAsync,
+    runEvent,
+    runWorker,
+    runWorkerAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/cjs/runtime/pyodide.js

@@ -0,0 +1,40 @@
+'use strict';
+const { fetchPaths, stdio } = require("./_utils.js");
+const {
+    run,
+    runAsync,
+    runEvent,
+    runWorker,
+    runWorkerAsync,
+    writeFile
+} = require("./_python.js");
+
+const type = "pyodide";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type: [type, "py"],
+    module: (version = "0.23.2") =>
+        `https://cdn.jsdelivr.net/pyodide/v${version}/full/pyodide.mjs`,
+    async engine({ loadPyodide }, config, url) {
+        const { stderr, stdout, get } = stdio();
+        const indexURL = url.slice(0, url.lastIndexOf("/"));
+        const runtime = await get(loadPyodide({ stderr, stdout, indexURL }));
+        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
+        if (config.packages) {
+            await runtime.loadPackage("micropip");
+            const micropip = await runtime.pyimport("micropip");
+            await micropip.install(config.packages);
+            micropip.destroy();
+        }
+        return runtime;
+    },
+    run,
+    runAsync,
+    runEvent,
+    runWorker,
+    runWorkerAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/cjs/runtime/ruby.js

@@ -0,0 +1,50 @@
+'use strict';
+const { clean, fetchPaths } = require("./_utils.js");
+
+const type = "ruby";
+
+// MISSING:
+//  * there is no VFS apparently or I couldn't reach any
+//  * I've no idea how to override the stderr and stdout
+//  * I've no idea how to import packages
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+const worker = (method) =>
+    function (runtime, code, xworker) {
+        globalThis.xworker = xworker;
+        return this[method](
+            runtime,
+            `require "js";xworker=JS::eval("return xworker");${code}`,
+        );
+    };
+
+module.exports = {
+    experimental: true,
+    type: [type, "rb"],
+    module: (version = "2.0.0") =>
+        `https://cdn.jsdelivr.net/npm/ruby-3_2-wasm-wasi@${version}/dist/browser.esm.js`,
+    async engine({ DefaultRubyVM }, config, url) {
+        const response = await fetch(
+            `${url.slice(0, url.lastIndexOf("/"))}/ruby.wasm`,
+        );
+        const module = await WebAssembly.compile(await response.arrayBuffer());
+        const { vm: runtime } = await DefaultRubyVM(module);
+        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
+        return runtime;
+    },
+    run: (runtime, code) => runtime.eval(clean(code)),
+    runAsync: (runtime, code) => runtime.evalAsync(clean(code)),
+    runEvent(runtime, code, key) {
+        return this.run(
+            runtime,
+            `require "js";event=JS::eval("return __events.get(${key})");${code}`,
+        );
+    },
+    runWorker: worker("run"),
+    runWorkerAsync: worker("runAsync"),
+    writeFile: () => {
+        throw new Error(`writeFile is not supported in ${type}`);
+    },
+};
+/* c8 ignore stop */