@pyscript/core 0.0.0

package/README.md

@@ -0,0 +1,33 @@
+# @pyscript/core
+
+[![build](https://github.com/WebReflection/python/actions/workflows/node.js.yml/badge.svg)](https://github.com/WebReflection/python/actions/workflows/node.js.yml) [![Coverage Status](https://coveralls.io/repos/github/WebReflection/python/badge.svg?branch=api&t=1RBdLX)](https://coveralls.io/github/WebReflection/python?branch=api)
+
+---
+
+## Development
+
+The working folder (source code of truth) is the `./esm` one, while the `./cjs` is populated as dual module and to test (but it's 1:1 code, no trnaspilation except for imports/exports).
+
+```sh
+# install all dependencies needed by core
+npm i
+```
+
+### Build / Artifacts
+
+This project requires some automatic artifact creation to:
+
+    * create a _Worker_ as a _Blob_ based on the same code used by this repo
+    * create automatically the list of runtimes available via the module
+    * create the `min.js` file used by most integration tests
+    * create a sha256 version of the Blob content for CSP cases
+
+Accordingly, to build latest project:
+
+```sh
+# create all artifacts needed to test core
+npm run build
+
+# optionally spin a server with CORS, COOP, and COEP enabled
+npm run server
+```

package/cjs/custom-types.js

@@ -0,0 +1,212 @@
+'use strict';
+require("@ungap/with-resolvers");
+const { $$ } = require("basic-devtools");
+
+const { assign, create } = require("./utils.js");
+const { getDetails } = require("./script-handler.js");
+const {
+    registry: defaultRegistry,
+    prefixes,
+    configs
+} = require("./interpreters.js");
+const { getRuntimeID } = require("./loader.js");
+const { io } = require("./interpreter/_utils.js");
+const { addAllListeners } = require("./listeners.js");
+
+const workerHooks = (m => /* c8 ignore start */ m.__esModule ? m.default : m /* c8 ignore stop */)(require("./worker/hooks.js"));
+
+const CUSTOM_SELECTORS = [];
+exports.CUSTOM_SELECTORS = CUSTOM_SELECTORS;
+
+/**
+ * @typedef {Object} Runtime custom configuration
+ * @prop {object} interpreter the bootstrapped interpreter
+ * @prop {(url:string, options?: object) => Worker} XWorker an XWorker constructor that defaults to same interpreter on the Worker.
+ * @prop {object} config a cloned config used to bootstrap the interpreter
+ * @prop {(code:string) => any} run an utility to run code within the interpreter
+ * @prop {(code:string) => Promise<any>} runAsync an utility to run code asynchronously within the interpreter
+ * @prop {(path:string, data:ArrayBuffer) => void} writeFile an utility to write a file in the virtual FS, if available
+ */
+
+const patched = new Map();
+const types = new Map();
+const waitList = new Map();
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+/**
+ * @param {Element} node any DOM element registered via define.
+ */
+const handleCustomType = (node) => {
+    for (const selector of CUSTOM_SELECTORS) {
+        if (node.matches(selector)) {
+            const type = types.get(selector);
+            const { resolve } = waitList.get(type);
+            const { options, known } = registry.get(type);
+            if (!known.has(node)) {
+                known.add(node);
+                const {
+                    interpreter: runtime,
+                    version,
+                    config,
+                    env,
+                    onRuntimeReady,
+                } = options;
+                const name = getRuntimeID(runtime, version);
+                const id = env || `${name}${config ? `|${config}` : ""}`;
+                const { interpreter: engine, XWorker } = getDetails(
+                    runtime,
+                    id,
+                    name,
+                    version,
+                    config,
+                );
+                engine.then((interpreter) => {
+                    if (!patched.has(id)) {
+                        const module = create(defaultRegistry.get(runtime));
+                        const {
+                            onBeforeRun,
+                            onBeforeRunAsync,
+                            onAfterRun,
+                            onAfterRunAsync,
+                            codeBeforeRunWorker,
+                            codeBeforeRunWorkerAsync,
+                            codeAfterRunWorker,
+                            codeAfterRunWorkerAsync,
+                        } = options;
+
+                        // These two loops mimic a `new Map(arrayContent)` without needing
+                        // the new Map overhead so that [name, [before, after]] can be easily destructured
+                        // and new sync or async patches become easy to add (when the logic is the same).
+
+                        // patch sync
+                        for (const [name, [before, after]] of [
+                            ["run", [onBeforeRun, onAfterRun]],
+                        ]) {
+                            const method = module[name];
+                            module[name] = function (interpreter, code) {
+                                if (before) before.call(this, resolved, node);
+                                const result = method.call(
+                                    this,
+                                    interpreter,
+                                    code,
+                                );
+                                if (after) after.call(this, resolved, node);
+                                return result;
+                            };
+                        }
+
+                        // patch async
+                        for (const [name, [before, after]] of [
+                            ["runAsync", [onBeforeRunAsync, onAfterRunAsync]],
+                        ]) {
+                            const method = module[name];
+                            module[name] = async function (interpreter, code) {
+                                if (before)
+                                    await before.call(this, resolved, node);
+                                const result = await method.call(
+                                    this,
+                                    interpreter,
+                                    code,
+                                );
+                                if (after)
+                                    await after.call(this, resolved, node);
+                                return result;
+                            };
+                        }
+
+                        // setup XWorker hooks, allowing strings to be forwarded to the worker
+                        // whenever it's created, as functions can't possibly be serialized
+                        // unless these are pure with no outer scope access (or globals vars)
+                        // so that making it strings disambiguate about their running context.
+                        workerHooks.set(XWorker, {
+                            beforeRun: codeBeforeRunWorker,
+                            beforeRunAsync: codeBeforeRunWorkerAsync,
+                            afterRun: codeAfterRunWorker,
+                            afterRunAsync: codeAfterRunWorkerAsync,
+                        });
+
+                        module.setGlobal(interpreter, "XWorker", XWorker);
+
+                        const resolved = {
+                            type,
+                            interpreter,
+                            XWorker,
+                            io: io.get(interpreter),
+                            config: structuredClone(configs.get(name)),
+                            run: module.run.bind(module, interpreter),
+                            runAsync: module.runAsync.bind(module, interpreter),
+                        };
+
+                        patched.set(id, resolved);
+                        resolve(resolved);
+                    }
+
+                    onRuntimeReady?.(patched.get(id), node);
+                });
+            }
+        }
+    }
+};
+exports.handleCustomType = handleCustomType;
+
+/**
+ * @type {Map<string, {options:object, known:WeakSet<Element>}>}
+ */
+const registry = new Map();
+
+/**
+ * @typedef {Object} PluginOptions custom configuration
+ * @prop {'pyodide' | 'micropython' | 'wasmoon' | 'ruby-wasm-wasi'} interpreter the interpreter to use
+ * @prop {string} [version] the optional interpreter version to use
+ * @prop {string} [config] the optional config to use within such interpreter
+ * @prop {(environment: object, node: Element) => void} [onRuntimeReady] the callback that will be invoked once
+ */
+
+/**
+ * Allows custom types and components on the page to receive interpreters to execute any code
+ * @param {string} type the unique `<script type="...">` identifier
+ * @param {PluginOptions} options the custom type configuration
+ */
+const define = (type, options) => {
+    if (defaultRegistry.has(type) || registry.has(type))
+        throw new Error(`<script type="${type}"> already registered`);
+
+    if (!defaultRegistry.has(options?.interpreter))
+        throw new Error(`Unspecified interpreter`);
+
+    // allows reaching out the interpreter helpers on events
+    defaultRegistry.set(type, defaultRegistry.get(options?.interpreter));
+
+    // ensure a Promise can resolve once a custom type has been bootstrapped
+    whenDefined(type);
+
+    // allows selector -> registry by type
+    const selectors = [`script[type="${type}"]`, `${type}-script`];
+    for (const selector of selectors) types.set(selector, type);
+
+    CUSTOM_SELECTORS.push(...selectors);
+    prefixes.push(`${type}-`);
+
+    // ensure always same env for this custom type
+    registry.set(type, {
+        options: assign({ env: type }, options),
+        known: new WeakSet(),
+    });
+
+    addAllListeners(document);
+    $$(selectors.join(",")).forEach(handleCustomType);
+};
+exports.define = define;
+
+/**
+ * Resolves whenever a defined custom type is bootstrapped on the page
+ * @param {string} type the unique `<script type="...">` identifier
+ * @returns {Promise<object>}
+ */
+const whenDefined = (type) => {
+    if (!waitList.has(type)) waitList.set(type, Promise.withResolvers());
+    return waitList.get(type).promise;
+};
+exports.whenDefined = whenDefined;
+/* c8 ignore stop */

package/cjs/fetch-utils.js

@@ -0,0 +1,12 @@
+'use strict';
+/** @param {Response} response */
+const getBuffer = (response) => response.arrayBuffer();
+exports.getBuffer = getBuffer;
+
+/** @param {Response} response */
+const getJSON = (response) => response.json();
+exports.getJSON = getJSON;
+
+/** @param {Response} response */
+const getText = (response) => response.text();
+exports.getText = getText;

package/cjs/index.js

@@ -0,0 +1,73 @@
+'use strict';
+const { $$ } = require("basic-devtools");
+
+const xworker = (m => /* c8 ignore start */ m.__esModule ? m.default : m /* c8 ignore stop */)(require("./worker/class.js"));
+const { handle } = require("./script-handler.js");
+const { assign } = require("./utils.js");
+const { selectors, prefixes } = require("./interpreters.js");
+const { CUSTOM_SELECTORS, handleCustomType } = require("./custom-types.js");
+const { listener, addAllListeners } = require("./listeners.js");
+
+(m => {
+  exports.define = m.define;
+  exports.whenDefined = m.whenDefined;
+})(require("./custom-types.js"));
+const XWorker = xworker();
+exports.XWorker = XWorker;
+
+const INTERPRETER_SELECTORS = selectors.join(",");
+
+const mo = new MutationObserver((records) => {
+    for (const { type, target, attributeName, addedNodes } of records) {
+        // attributes are tested via integration / e2e
+        /* c8 ignore next 17 */
+        if (type === "attributes") {
+            const i = attributeName.lastIndexOf("-") + 1;
+            if (i) {
+                const prefix = attributeName.slice(0, i);
+                for (const p of prefixes) {
+                    if (prefix === p) {
+                        const type = attributeName.slice(i);
+                        if (type !== "env") {
+                            const method = target.hasAttribute(attributeName)
+                                ? "add"
+                                : "remove";
+                            target[`${method}EventListener`](type, listener);
+                        }
+                        break;
+                    }
+                }
+            }
+            continue;
+        }
+        for (const node of addedNodes) {
+            if (node.nodeType === 1) {
+                addAllListeners(node);
+                if (node.matches(INTERPRETER_SELECTORS)) handle(node);
+                else {
+                    $$(INTERPRETER_SELECTORS, node).forEach(handle);
+                    if (!CUSTOM_SELECTORS.length) continue;
+                    handleCustomType(node);
+                    $$(CUSTOM_SELECTORS.join(","), node).forEach(
+                        handleCustomType,
+                    );
+                }
+            }
+        }
+    }
+});
+
+const observe = (root) => {
+    mo.observe(root, { childList: true, subtree: true, attributes: true });
+    return root;
+};
+
+const { attachShadow } = Element.prototype;
+assign(Element.prototype, {
+    attachShadow(init) {
+        return observe(attachShadow.call(this, init));
+    },
+});
+
+addAllListeners(observe(document));
+$$(INTERPRETER_SELECTORS, document).forEach(handle);

package/cjs/interpreter/_python.js

@@ -0,0 +1,16 @@
+'use strict';
+const { clean, writeFile: writeFileUtil } = require("./_utils.js");
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+const run = (interpreter, code) => interpreter.runPython(clean(code));
+exports.run = run;
+
+const runAsync = (interpreter, code) =>
+    interpreter.runPythonAsync(clean(code));
+exports.runAsync = runAsync;
+
+const writeFile = ({ FS }, path, buffer) =>
+    writeFileUtil(FS, path, buffer);
+exports.writeFile = writeFile;
+/* c8 ignore stop */

package/cjs/interpreter/_utils.js

@@ -0,0 +1,141 @@
+'use strict';
+require("@ungap/with-resolvers");
+
+const { getBuffer } = require("../fetch-utils.js");
+const { absoluteURL } = require("../utils.js");
+
+/**
+ * Trim code only if it's a single line that prettier or other tools might have modified.
+ * @param {string} code code that might be a single line
+ * @returns {string}
+ */
+const clean = (code) =>
+    code.replace(/^[^\r\n]+$/, (line) => line.trim());
+exports.clean = clean;
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+const io = new WeakMap();
+exports.io = io;
+const stdio = (init) => {
+    const context = init || console;
+    const localIO = {
+        stderr: (context.stderr || console.error).bind(context),
+        stdout: (context.stdout || console.log).bind(context),
+    };
+    return {
+        stderr: (...args) => localIO.stderr(...args),
+        stdout: (...args) => localIO.stdout(...args),
+        async get(engine) {
+            const interpreter = await engine;
+            io.set(interpreter, localIO);
+            return interpreter;
+        },
+    };
+};
+exports.stdio = stdio;
+/* c8 ignore stop */
+
+// This should be the only helper needed for all Emscripten based FS exports
+const writeFile = (FS, path, buffer) => {
+    const { parentPath, name } = FS.analyzePath(path, true);
+    FS.mkdirTree(parentPath);
+    return FS.writeFile([parentPath, name].join("/"), new Uint8Array(buffer), {
+        canOwn: true,
+    });
+};
+exports.writeFile = writeFile;
+
+// This is instead a fallback for Lua or others
+const writeFileShim = (FS, path, buffer) => {
+    path = resolve(FS, path);
+    mkdirTree(FS, dirname(path));
+    return FS.writeFile(path, new Uint8Array(buffer), { canOwn: true });
+};
+exports.writeFileShim = writeFileShim;
+
+const dirname = (path) => {
+    const tree = path.split("/");
+    tree.pop();
+    return tree.join("/");
+};
+
+const mkdirTree = (FS, path) => {
+    const current = [];
+    for (const branch of path.split("/")) {
+        current.push(branch);
+        if (branch) FS.mkdir(current.join("/"));
+    }
+};
+
+const resolve = (FS, path) => {
+    const tree = [];
+    for (const branch of path.split("/")) {
+        switch (branch) {
+            case "":
+                break;
+            case ".":
+                break;
+            case "..":
+                tree.pop();
+                break;
+            default:
+                tree.push(branch);
+        }
+    }
+    return [FS.cwd()].concat(tree).join("/").replace(/^\/+/, "/");
+};
+
+const { all, isArray } = require("../utils.js");
+
+const calculateFetchPaths = (config_fetch) => {
+    // REQUIRES INTEGRATION TEST
+    /* c8 ignore start */
+    for (const { files, to_file, from = "" } of config_fetch) {
+        if (files !== undefined && to_file !== undefined)
+            throw new Error(
+                `Cannot use 'to_file' and 'files' parameters together!`,
+            );
+        if (files === undefined && to_file === undefined && from.endsWith("/"))
+            throw new Error(
+                `Couldn't determine the filename from the path ${from}, please supply 'to_file' parameter.`,
+            );
+    }
+    /* c8 ignore stop */
+    return config_fetch.flatMap(
+        ({ from = "", to_folder = ".", to_file, files }) => {
+            if (isArray(files))
+                return files.map((file) => ({
+                    url: joinPaths([from, file]),
+                    path: joinPaths([to_folder, file]),
+                }));
+            const filename = to_file || from.slice(1 + from.lastIndexOf("/"));
+            return [{ url: from, path: joinPaths([to_folder, filename]) }];
+        },
+    );
+};
+
+const joinPaths = (parts) => {
+    const res = parts
+        .map((part) => part.trim().replace(/(^[/]*|[/]*$)/g, ""))
+        .filter((p) => p !== "" && p !== ".")
+        .join("/");
+
+    return parts[0].startsWith("/") ? `/${res}` : res;
+};
+
+const fetchResolved = (config_fetch, url) =>
+    fetch(absoluteURL(url, base.get(config_fetch)));
+
+const base = new WeakMap();
+exports.base = base;
+
+const fetchPaths = (module, interpreter, config_fetch) =>
+    all(
+        calculateFetchPaths(config_fetch).map(({ url, path }) =>
+            fetchResolved(config_fetch, url)
+                .then(getBuffer)
+                .then((buffer) => module.writeFile(interpreter, path, buffer)),
+        ),
+    );
+exports.fetchPaths = fetchPaths;

package/cjs/interpreter/micropython.js

@@ -0,0 +1,34 @@
+'use strict';
+const { fetchPaths, stdio } = require("./_utils.js");
+const { run, runAsync, writeFile } = require("./_python.js");
+
+const type = "micropython";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type,
+    module: (version = "1.20.0-239") =>
+        `https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript@${version}/micropython.mjs`,
+    async engine({ loadMicroPython }, config, url) {
+        const { stderr, stdout, get } = stdio();
+        url = url.replace(/\.m?js$/, ".wasm");
+        const runtime = await get(loadMicroPython({ stderr, stdout, url }));
+        if (config.fetch) await fetchPaths(this, runtime, config.fetch);
+        return runtime;
+    },
+    setGlobal(interpreter, name, value) {
+        const id = `__pyscript_${this.type}_${name}`;
+        globalThis[id] = value;
+        this.run(interpreter, `from js import ${id};${name}=${id};`);
+    },
+    deleteGlobal(interpreter, name) {
+        const id = `__pyscript_${this.type}_${name}`;
+        this.run(interpreter, `del ${id};del ${name}`);
+        delete globalThis[id];
+    },
+    run,
+    runAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/cjs/interpreter/pyodide.js

@@ -0,0 +1,38 @@
+'use strict';
+const { fetchPaths, stdio } = require("./_utils.js");
+const { run, runAsync, writeFile } = require("./_python.js");
+
+const type = "pyodide";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type,
+    module: (version = "0.23.2") =>
+        `https://cdn.jsdelivr.net/pyodide/v${version}/full/pyodide.mjs`,
+    async engine({ loadPyodide }, config, url) {
+        const { stderr, stdout, get } = stdio();
+        const indexURL = url.slice(0, url.lastIndexOf("/"));
+        const interpreter = await get(
+            loadPyodide({ stderr, stdout, indexURL }),
+        );
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.packages) {
+            await interpreter.loadPackage("micropip");
+            const micropip = await interpreter.pyimport("micropip");
+            await micropip.install(config.packages);
+            micropip.destroy();
+        }
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        interpreter.globals.set(name, value);
+    },
+    deleteGlobal(interpreter, name) {
+        interpreter.globals.delete(name);
+    },
+    run,
+    runAsync,
+    writeFile,
+};
+/* c8 ignore stop */

package/cjs/interpreter/ruby-wasm-wasi.js

@@ -0,0 +1,43 @@
+'use strict';
+const { clean, fetchPaths } = require("./_utils.js");
+
+const type = "ruby-wasm-wasi";
+
+// MISSING:
+//  * there is no VFS apparently or I couldn't reach any
+//  * I've no idea how to override the stderr and stdout
+//  * I've no idea how to import packages
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type,
+    experimental: true,
+    module: (version = "2.0.0") =>
+        `https://cdn.jsdelivr.net/npm/ruby-3_2-wasm-wasi@${version}/dist/browser.esm.js`,
+    async engine({ DefaultRubyVM }, config, url) {
+        const response = await fetch(
+            `${url.slice(0, url.lastIndexOf("/"))}/ruby.wasm`,
+        );
+        const module = await WebAssembly.compile(await response.arrayBuffer());
+        const { vm: interpreter } = await DefaultRubyVM(module);
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        const id = `__pyscript_ruby_wasm_wasi_${name}`;
+        globalThis[id] = value;
+        this.run(interpreter, `require "js";$${name}=JS::eval("return ${id}")`);
+    },
+    deleteGlobal(interpreter, name) {
+        const id = `__pyscript_ruby_wasm_wasi_${name}`;
+        this.run(interpreter, `$${name}=nil`);
+        delete globalThis[id];
+    },
+    run: (interpreter, code) => interpreter.eval(clean(code)),
+    runAsync: (interpreter, code) => interpreter.evalAsync(clean(code)),
+    writeFile: () => {
+        throw new Error(`writeFile is not supported in ${type}`);
+    },
+};
+/* c8 ignore stop */

package/cjs/interpreter/wasmoon.js

@@ -0,0 +1,40 @@
+'use strict';
+const { clean, fetchPaths, stdio, writeFileShim } = require("./_utils.js");
+
+const type = "wasmoon";
+
+// REQUIRES INTEGRATION TEST
+/* c8 ignore start */
+module.exports = {
+    type,
+    module: (version = "1.15.0") =>
+        `https://cdn.jsdelivr.net/npm/wasmoon@${version}/+esm`,
+    async engine({ LuaFactory, LuaLibraries }, config) {
+        const { stderr, stdout, get } = stdio();
+        const interpreter = await get(new LuaFactory().createEngine());
+        interpreter.global.getTable(LuaLibraries.Base, (index) => {
+            interpreter.global.setField(index, "print", stdout);
+            interpreter.global.setField(index, "printErr", stderr);
+        });
+        if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        return interpreter;
+    },
+    setGlobal(interpreter, name, value) {
+        interpreter.global.set(name, value);
+    },
+    deleteGlobal(interpreter, name) {
+        interpreter.global.set(name, void 0);
+    },
+    run: (interpreter, code) => interpreter.doStringSync(clean(code)),
+    runAsync: (interpreter, code) => interpreter.doString(clean(code)),
+    writeFile: (
+        {
+            cmodule: {
+                module: { FS },
+            },
+        },
+        path,
+        buffer,
+    ) => writeFileShim(FS, path, buffer),
+};
+/* c8 ignore stop */