@pyscript/core 0.0.0

package/esm/custom-types.js

@@ -0,0 +1,207 @@
+import "@ungap/with-resolvers";
+import { $$ } from "basic-devtools";
+
+import { assign, create } from "./utils.js";
+import { getDetails } from "./script-handler.js";
+import {
+    registry as defaultRegistry,
+    prefixes,
+    configs,
+} from "./interpreters.js";
+import { getRuntimeID } from "./loader.js";
+import { io } from "./interpreter/_utils.js";
+import { addAllListeners } from "./listeners.js";
+
+import workerHooks from "./worker/hooks.js";
+
+export const CUSTOM_SELECTORS = [];
+
+/**
+ * @typedef {Object} Runtime custom configuration
+ * @prop {object} interpreter the bootstrapped interpreter
+ * @prop {(url:string, options?: object) => Worker} XWorker an XWorker constructor that defaults to same interpreter on the Worker.
+ * @prop {object} config a cloned config used to bootstrap the interpreter
+ * @prop {(code:string) => any} run an utility to run code within the interpreter
+ * @prop {(code:string) => Promise<any>} runAsync an utility to run code asynchronously within the interpreter
+ * @prop {(path:string, data:ArrayBuffer) => void} writeFile an utility to write a file in the virtual FS, if available
+ */
+
+const patched = new Map();
+const types = new Map();
+const waitList = new Map();
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+/**
+ * @param {Element} node any DOM element registered via define.
+ */
+export const handleCustomType = (node) => {
+    for (const selector of CUSTOM_SELECTORS) {
+        if (node.matches(selector)) {
+            const type = types.get(selector);
+            const { resolve } = waitList.get(type);
+            const { options, known } = registry.get(type);
+            if (!known.has(node)) {
+                known.add(node);
+                const {
+                    interpreter: runtime,
+                    version,
+                    config,
+                    env,
+                    onRuntimeReady,
+                } = options;
+                const name = getRuntimeID(runtime, version);
+                const id = env || `${name}${config ? `|${config}` : ""}`;
+                const { interpreter: engine, XWorker } = getDetails(
+                    runtime,
+                    id,
+                    name,
+                    version,
+                    config,
+                );
+                engine.then((interpreter) => {
+                    if (!patched.has(id)) {
+                        const module = create(defaultRegistry.get(runtime));
+                        const {
+                            onBeforeRun,
+                            onBeforeRunAsync,
+                            onAfterRun,
+                            onAfterRunAsync,
+                            codeBeforeRunWorker,
+                            codeBeforeRunWorkerAsync,
+                            codeAfterRunWorker,
+                            codeAfterRunWorkerAsync,
+                        } = options;
+
+                        // These two loops mimic a `new Map(arrayContent)` without needing
+                        // the new Map overhead so that [name, [before, after]] can be easily destructured
+                        // and new sync or async patches become easy to add (when the logic is the same).
+
+                        // patch sync
+                        for (const [name, [before, after]] of [
+                            ["run", [onBeforeRun, onAfterRun]],
+                        ]) {
+                            const method = module[name];
+                            module[name] = function (interpreter, code) {
+                                if (before) before.call(this, resolved, node);
+                                const result = method.call(
+                                    this,
+                                    interpreter,
+                                    code,
+                                );
+                                if (after) after.call(this, resolved, node);
+                                return result;
+                            };
+                        }
+
+                        // patch async
+                        for (const [name, [before, after]] of [
+                            ["runAsync", [onBeforeRunAsync, onAfterRunAsync]],
+                        ]) {
+                            const method = module[name];
+                            module[name] = async function (interpreter, code) {
+                                if (before)
+                                    await before.call(this, resolved, node);
+                                const result = await method.call(
+                                    this,
+                                    interpreter,
+                                    code,
+                                );
+                                if (after)
+                                    await after.call(this, resolved, node);
+                                return result;
+                            };
+                        }
+
+                        // setup XWorker hooks, allowing strings to be forwarded to the worker
+                        // whenever it's created, as functions can't possibly be serialized
+                        // unless these are pure with no outer scope access (or globals vars)
+                        // so that making it strings disambiguate about their running context.
+                        workerHooks.set(XWorker, {
+                            beforeRun: codeBeforeRunWorker,
+                            beforeRunAsync: codeBeforeRunWorkerAsync,
+                            afterRun: codeAfterRunWorker,
+                            afterRunAsync: codeAfterRunWorkerAsync,
+                        });
+
+                        module.setGlobal(interpreter, "XWorker", XWorker);
+
+                        const resolved = {
+                            type,
+                            interpreter,
+                            XWorker,
+                            io: io.get(interpreter),
+                            config: structuredClone(configs.get(name)),
+                            run: module.run.bind(module, interpreter),
+                            runAsync: module.runAsync.bind(module, interpreter),
+                        };
+
+                        patched.set(id, resolved);
+                        resolve(resolved);
+                    }
+
+                    onRuntimeReady?.(patched.get(id), node);
+                });
+            }
+        }
+    }
+};
+
+/**
+ * @type {Map<string, {options:object, known:WeakSet<Element>}>}
+ */
+const registry = new Map();
+
+/**
+ * @typedef {Object} PluginOptions custom configuration
+ * @prop {'pyodide' | 'micropython' | 'wasmoon' | 'ruby-wasm-wasi'} interpreter the interpreter to use
+ * @prop {string} [version] the optional interpreter version to use
+ * @prop {string} [config] the optional config to use within such interpreter
+ * @prop {(environment: object, node: Element) => void} [onRuntimeReady] the callback that will be invoked once
+ */
+
+/**
+ * Allows custom types and components on the page to receive interpreters to execute any code
+ * @param {string} type the unique `<script type="...">` identifier
+ * @param {PluginOptions} options the custom type configuration
+ */
+export const define = (type, options) => {
+    if (defaultRegistry.has(type) || registry.has(type))
+        throw new Error(`<script type="${type}"> already registered`);
+
+    if (!defaultRegistry.has(options?.interpreter))
+        throw new Error(`Unspecified interpreter`);
+
+    // allows reaching out the interpreter helpers on events
+    defaultRegistry.set(type, defaultRegistry.get(options?.interpreter));
+
+    // ensure a Promise can resolve once a custom type has been bootstrapped
+    whenDefined(type);
+
+    // allows selector -> registry by type
+    const selectors = [`script[type="${type}"]`, `${type}-script`];
+    for (const selector of selectors) types.set(selector, type);
+
+    CUSTOM_SELECTORS.push(...selectors);
+    prefixes.push(`${type}-`);
+
+    // ensure always same env for this custom type
+    registry.set(type, {
+        options: assign({ env: type }, options),
+        known: new WeakSet(),
+    });
+
+    addAllListeners(document);
+    $$(selectors.join(",")).forEach(handleCustomType);
+};
+
+/**
+ * Resolves whenever a defined custom type is bootstrapped on the page
+ * @param {string} type the unique `<script type="...">` identifier
+ * @returns {Promise<object>}
+ */
+export const whenDefined = (type) => {
+    if (!waitList.has(type)) waitList.set(type, Promise.withResolvers());
+    return waitList.get(type).promise;
+};
+/* c8 ignore stop */

package/esm/fetch-utils.js

@@ -0,0 +1,8 @@
+/** @param {Response} response */
+export const getBuffer = (response) => response.arrayBuffer();
+
+/** @param {Response} response */
+export const getJSON = (response) => response.json();
+
+/** @param {Response} response */
+export const getText = (response) => response.text();

package/esm/index.js

@@ -0,0 +1,68 @@
+import { $$ } from "basic-devtools";
+
+import xworker from "./worker/class.js";
+import { handle } from "./script-handler.js";
+import { assign } from "./utils.js";
+import { selectors, prefixes } from "./interpreters.js";
+import { CUSTOM_SELECTORS, handleCustomType } from "./custom-types.js";
+import { listener, addAllListeners } from "./listeners.js";
+
+export { define, whenDefined } from "./custom-types.js";
+export const XWorker = xworker();
+
+const INTERPRETER_SELECTORS = selectors.join(",");
+
+const mo = new MutationObserver((records) => {
+    for (const { type, target, attributeName, addedNodes } of records) {
+        // attributes are tested via integration / e2e
+        /* c8 ignore next 17 */
+        if (type === "attributes") {
+            const i = attributeName.lastIndexOf("-") + 1;
+            if (i) {
+                const prefix = attributeName.slice(0, i);
+                for (const p of prefixes) {
+                    if (prefix === p) {
+                        const type = attributeName.slice(i);
+                        if (type !== "env") {
+                            const method = target.hasAttribute(attributeName)
+                                ? "add"
+                                : "remove";
+                            target[`${method}EventListener`](type, listener);
+                        }
+                        break;
+                    }
+                }
+            }
+            continue;
+        }
+        for (const node of addedNodes) {
+            if (node.nodeType === 1) {
+                addAllListeners(node);
+                if (node.matches(INTERPRETER_SELECTORS)) handle(node);
+                else {
+                    $$(INTERPRETER_SELECTORS, node).forEach(handle);
+                    if (!CUSTOM_SELECTORS.length) continue;
+                    handleCustomType(node);
+                    $$(CUSTOM_SELECTORS.join(","), node).forEach(
+                        handleCustomType,
+                    );
+                }
+            }
+        }
+    }
+});
+
+const observe = (root) => {
+    mo.observe(root, { childList: true, subtree: true, attributes: true });
+    return root;
+};
+
+const { attachShadow } = Element.prototype;
+assign(Element.prototype, {
+    attachShadow(init) {
+        return observe(attachShadow.call(this, init));
+    },
+});
+
+addAllListeners(observe(document));
+$$(INTERPRETER_SELECTORS, document).forEach(handle);

package/esm/interpreter/_python.js

@@ -0,0 +1,12 @@
+import { clean, writeFile as writeFileUtil } from "./_utils.js";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export const run = (interpreter, code) => interpreter.runPython(clean(code));
+
+export const runAsync = (interpreter, code) =>
+    interpreter.runPythonAsync(clean(code));
+
+export const writeFile = ({ FS }, path, buffer) =>
+    writeFileUtil(FS, path, buffer);
+/* c8 ignore stop */

package/esm/interpreter/_utils.js

@@ -0,0 +1,133 @@
+import "@ungap/with-resolvers";
+
+import { getBuffer } from "../fetch-utils.js";
+import { absoluteURL } from "../utils.js";
+
+/**
+ * Trim code only if it's a single line that prettier or other tools might have modified.
+ * @param {string} code code that might be a single line
+ * @returns {string}
+ */
+export const clean = (code) =>
+    code.replace(/^[^\r\n]+$/, (line) => line.trim());
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export const io = new WeakMap();
+export const stdio = (init) => {
+    const context = init || console;
+    const localIO = {
+        stderr: (context.stderr || console.error).bind(context),
+        stdout: (context.stdout || console.log).bind(context),
+    };
+    return {
+        stderr: (...args) => localIO.stderr(...args),
+        stdout: (...args) => localIO.stdout(...args),
+        async get(engine) {
+            const interpreter = await engine;
+            io.set(interpreter, localIO);
+            return interpreter;
+        },
+    };
+};
+/* c8 ignore stop */
+
+// This should be the only helper needed for all Emscripten based FS exports
+export const writeFile = (FS, path, buffer) => {
+    const { parentPath, name } = FS.analyzePath(path, true);
+    FS.mkdirTree(parentPath);
+    return FS.writeFile([parentPath, name].join("/"), new Uint8Array(buffer), {
+        canOwn: true,
+    });
+};
+
+// This is instead a fallback for Lua or others
+export const writeFileShim = (FS, path, buffer) => {
+    path = resolve(FS, path);
+    mkdirTree(FS, dirname(path));
+    return FS.writeFile(path, new Uint8Array(buffer), { canOwn: true });
+};
+
+const dirname = (path) => {
+    const tree = path.split("/");
+    tree.pop();
+    return tree.join("/");
+};
+
+const mkdirTree = (FS, path) => {
+    const current = [];
+    for (const branch of path.split("/")) {
+        current.push(branch);
+        if (branch) FS.mkdir(current.join("/"));
+    }
+};
+
+const resolve = (FS, path) => {
+    const tree = [];
+    for (const branch of path.split("/")) {
+        switch (branch) {
+            case "":
+                break;
+            case ".":
+                break;
+            case "..":
+                tree.pop();
+                break;
+            default:
+                tree.push(branch);
+        }
+    }
+    return [FS.cwd()].concat(tree).join("/").replace(/^\/+/, "/");
+};
+
+import { all, isArray } from "../utils.js";
+
+const calculateFetchPaths = (config_fetch) => {
+    // REQUIRES INTEGRATION TEST
+    /* c8 ignore start */
+    for (const { files, to_file, from = "" } of config_fetch) {
+        if (files !== undefined && to_file !== undefined)
+            throw new Error(
+                `Cannot use 'to_file' and 'files' parameters together!`,
+            );
+        if (files === undefined && to_file === undefined && from.endsWith("/"))
+            throw new Error(
+                `Couldn't determine the filename from the path ${from}, please supply 'to_file' parameter.`,
+            );
+    }
+    /* c8 ignore stop */
+    return config_fetch.flatMap(
+        ({ from = "", to_folder = ".", to_file, files }) => {
+            if (isArray(files))
+                return files.map((file) => ({
+                    url: joinPaths([from, file]),
+                    path: joinPaths([to_folder, file]),
+                }));
+            const filename = to_file || from.slice(1 + from.lastIndexOf("/"));
+            return [{ url: from, path: joinPaths([to_folder, filename]) }];
+        },
+    );
+};
+
+const joinPaths = (parts) => {
+    const res = parts
+        .map((part) => part.trim().replace(/(^[/]*|[/]*$)/g, ""))
+        .filter((p) => p !== "" && p !== ".")
+        .join("/");
+
+    return parts[0].startsWith("/") ? `/${res}` : res;
+};
+
+const fetchResolved = (config_fetch, url) =>
+    fetch(absoluteURL(url, base.get(config_fetch)));
+
+export const base = new WeakMap();
+
+export const fetchPaths = (module, interpreter, config_fetch) =>
+    all(
+        calculateFetchPaths(config_fetch).map(({ url, path }) =>
+            fetchResolved(config_fetch, url)
+                .then(getBuffer)
+                .then((buffer) => module.writeFile(interpreter, path, buffer)),
+        ),
+    );

package/esm/interpreter/micropython.js

@@ -0,0 +1,33 @@
+import { fetchPaths, stdio } from "./_utils.js";
+import { run, runAsync, writeFile } from "./_python.js";
+
+const type = "micropython";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export default {
+    type,
+    module: (version = "1.20.0-239") =>
+        `https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript@${version}/micropython.mjs`,
+    async engine({ loadMicroPython }, config, url) {
+        const { stderr, stdout, get } = stdio();
+        url = url.replace(/\.m?js$/, ".wasm");
+        const runtime = await get(loadMicroPython({ stderr, stdout, url }));
+        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
+        return runtime;
+    },
+    setGlobal(interpreter, name, value) {
+        const id = `__pyscript_${this.type}_${name}`;
+        globalThis[id] = value;
+        this.run(interpreter, `from js import ${id};${name}=${id};`);
+    },
+    deleteGlobal(interpreter, name) {
+        const id = `__pyscript_${this.type}_${name}`;
+        this.run(interpreter, `del ${id};del ${name}`);
+        delete globalThis[id];
+    },
+    run,
+    runAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/esm/interpreter/pyodide.js

@@ -0,0 +1,37 @@
+import { fetchPaths, stdio } from "./_utils.js";
+import { run, runAsync, writeFile } from "./_python.js";
+
+const type = "pyodide";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export default {
+    type,
+    module: (version = "0.23.2") =>
+        `https://cdn.jsdelivr.net/pyodide/v${version}/full/pyodide.mjs`,
+    async engine({ loadPyodide }, config, url) {
+        const { stderr, stdout, get } = stdio();
+        const indexURL = url.slice(0, url.lastIndexOf("/"));
+        const interpreter = await get(
+            loadPyodide({ stderr, stdout, indexURL }),
+        );
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.packages) {
+            await interpreter.loadPackage("micropip");
+            const micropip = await interpreter.pyimport("micropip");
+            await micropip.install(config.packages);
+            micropip.destroy();
+        }
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        interpreter.globals.set(name, value);
+    },
+    deleteGlobal(interpreter, name) {
+        interpreter.globals.delete(name);
+    },
+    run,
+    runAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/esm/interpreter/ruby-wasm-wasi.js

@@ -0,0 +1,42 @@
+import { clean, fetchPaths } from "./_utils.js";
+
+const type = "ruby-wasm-wasi";
+
+// MISSING:
+//  * there is no VFS apparently or I couldn't reach any
+//  * I've no idea how to override the stderr and stdout
+//  * I've no idea how to import packages
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export default {
+    type,
+    experimental: true,
+    module: (version = "2.0.0") =>
+        `https://cdn.jsdelivr.net/npm/ruby-3_2-wasm-wasi@${version}/dist/browser.esm.js`,
+    async engine({ DefaultRubyVM }, config, url) {
+        const response = await fetch(
+            `${url.slice(0, url.lastIndexOf("/"))}/ruby.wasm`,
+        );
+        const module = await WebAssembly.compile(await response.arrayBuffer());
+        const { vm: interpreter } = await DefaultRubyVM(module);
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        const id = `__pyscript_ruby_wasm_wasi_${name}`;
+        globalThis[id] = value;
+        this.run(interpreter, `require "js";$${name}=JS::eval("return ${id}")`);
+    },
+    deleteGlobal(interpreter, name) {
+        const id = `__pyscript_ruby_wasm_wasi_${name}`;
+        this.run(interpreter, `$${name}=nil`);
+        delete globalThis[id];
+    },
+    run: (interpreter, code) => interpreter.eval(clean(code)),
+    runAsync: (interpreter, code) => interpreter.evalAsync(clean(code)),
+    writeFile: () => {
+        throw new Error(`writeFile is not supported in ${type}`);
+    },
+};
+/* c8 ignore stop */

package/esm/interpreter/wasmoon.js

@@ -0,0 +1,39 @@
+import { clean, fetchPaths, stdio, writeFileShim } from "./_utils.js";
+
+const type = "wasmoon";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+export default {
+    type,
+    module: (version = "1.15.0") =>
+        `https://cdn.jsdelivr.net/npm/wasmoon@${version}/+esm`,
+    async engine({ LuaFactory, LuaLibraries }, config) {
+        const { stderr, stdout, get } = stdio();
+        const interpreter = await get(new LuaFactory().createEngine());
+        interpreter.global.getTable(LuaLibraries.Base, (index) => {
+            interpreter.global.setField(index, "print", stdout);
+            interpreter.global.setField(index, "printErr", stderr);
+        });
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        interpreter.global.set(name, value);
+    },
+    deleteGlobal(interpreter, name) {
+        interpreter.global.set(name, void 0);
+    },
+    run: (interpreter, code) => interpreter.doStringSync(clean(code)),
+    runAsync: (interpreter, code) => interpreter.doString(clean(code)),
+    writeFile: (
+        {
+            cmodule: {
+                module: { FS },
+            },
+        },
+        path,
+        buffer,
+    ) => writeFileShim(FS, path, buffer),
+};
+/* c8 ignore stop */

package/esm/interpreters.js

@@ -0,0 +1,58 @@
+// ⚠️ Part of this file is automatically generated
+//    The :RUNTIMES comment is a delimiter and no code should be written/changed after
+//    See rollup/build_interpreters.cjs to know more
+
+import { base } from "./interpreter/_utils.js";
+
+/** @type {Map<string, object>} */
+export const registry = new Map();
+
+/** @type {Map<string, object>} */
+export const configs = new Map();
+
+/** @type {string[]} */
+export const selectors = [];
+
+/** @type {string[]} */
+export const prefixes = [];
+
+export const interpreter = new Proxy(new Map(), {
+    get(map, id) {
+        if (!map.has(id)) {
+            const [type, ...rest] = id.split("@");
+            const interpreter = registry.get(type);
+            const url = /^https?:\/\//i.test(rest)
+                ? rest.join("@")
+                : interpreter.module(...rest);
+            map.set(id, {
+                url,
+                module: import(url),
+                engine: interpreter.engine.bind(interpreter),
+            });
+        }
+        const { url, module, engine } = map.get(id);
+        return (config, baseURL) =>
+            module.then((module) => {
+                configs.set(id, config);
+                const fetch = config?.fetch;
+                if (fetch) base.set(fetch, baseURL);
+                return engine(module, config, url);
+            });
+    },
+});
+
+const register = (interpreter) => {
+    for (const type of [].concat(interpreter.type)) {
+        registry.set(type, interpreter);
+        selectors.push(`script[type="${type}"]`);
+        prefixes.push(`${type}-`);
+    }
+};
+
+//:RUNTIMES
+import micropython from "./interpreter/micropython.js";
+import pyodide from "./interpreter/pyodide.js";
+import ruby_wasm_wasi from "./interpreter/ruby-wasm-wasi.js";
+import wasmoon from "./interpreter/wasmoon.js";
+for (const interpreter of [micropython, pyodide, ruby_wasm_wasi, wasmoon])
+    register(interpreter);