@stlite/desktop 0.18.2 → 0.19.0

package/bin/dump_artifacts.js

@@ -1,186 +1,13 @@
 #!/usr/bin/env node
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const yargs_1 = __importDefault(require("yargs"));
-const helpers_1 = require("yargs/helpers");
-const path_1 = __importDefault(require("path"));
-const promises_1 = __importDefault(require("fs/promises"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const node_fetch_1 = __importDefault(require("node-fetch"));
-const pyodide_1 = require("pyodide");
-// @ts-ignore
-global.fetch = node_fetch_1.default; // The global `fetch()` is necessary for micropip.install() to load the remote packages.
-async function copyBuildDirectory(options) {
-    console.info("Copy the build directory (the bare built app files) to this directory...");
-    const sourceDir = path_1.default.resolve(__dirname, "../build");
-    const sourceDirStat = await promises_1.default.stat(sourceDir);
-    if (!sourceDirStat.isDirectory()) {
-        throw new Error(`The source ${sourceDir} does not exist.`);
-    }
-    if (sourceDir === options.copyTo) {
-        console.warn(`sourceDir == destDir (${sourceDir}). Are you in the development environment? Skip copying the directory.`);
-        return;
-    }
-    if (options.keepOld) {
-        try {
-            await promises_1.default.access(options.copyTo);
-            console.info(`${options.copyTo} already exists. Use it and skip copying.`);
-            return;
-        }
-        catch {
-            // If the destination directory does not exist
-            throw new Error(`${options.copyTo} does not exist even though the \`keepOld\` option is specified`);
-        }
-    }
-    console.log(`Copy ${sourceDir} to ${options.copyTo}`);
-    await promises_1.default.rm(options.copyTo, { recursive: true, force: true });
-    await fs_extra_1.default.copy(sourceDir, options.copyTo);
-}
-async function installLocalWheel(pyodide, localPath) {
-    console.log(`Install the local wheel ${localPath}`);
-    const data = await promises_1.default.readFile(localPath);
-    const emfsPath = "/tmp/" + path_1.default.basename(localPath);
-    pyodide.FS.writeFile(emfsPath, data);
-    const micropip = pyodide.pyimport("micropip");
-    const requirement = `emfs:${emfsPath}`;
-    console.log(`Install ${requirement}`);
-    await micropip.install.callKwargs(requirement, { keep_going: true });
-}
-async function createSitePackagesSnapshot(options) {
-    console.info("Create the site-packages snapshot file...");
-    const pyodide = await (0, pyodide_1.loadPyodide)();
-    await pyodide.loadPackage(["micropip"]);
-    if (options.useLocalKernelWheels) {
-        const stliteKernelDir = path_1.default.dirname(require.resolve("@stlite/kernel")); // -> /path/to/kernel/dist
-        const stliteKernelPyDir = path_1.default.resolve(stliteKernelDir, "../py"); // -> /path/to/kernel/py
-        await installLocalWheel(pyodide, path_1.default.join(stliteKernelPyDir, "tornado/dist/tornado-6.2-py3-none-any.whl"));
-        await installLocalWheel(pyodide, path_1.default.join(stliteKernelPyDir, "stlite-pyarrow/dist/stlite_pyarrow-0.1.0-py3-none-any.whl"));
-        await installLocalWheel(pyodide, path_1.default.join(stliteKernelPyDir, "streamlit/lib/dist/streamlit-1.13.0-py2.py3-none-any.whl"));
-    }
-    else {
-        const micropip = pyodide.pyimport("micropip");
-        const wheelUrls = [
-            "https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/tornado/dist/tornado-6.2-py3-none-any.whl",
-            "https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/stlite-pyarrow/dist/stlite_pyarrow-0.1.0-py3-none-any.whl",
-            "https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/streamlit/lib/dist/streamlit-1.13.0-py2.py3-none-any.whl",
-        ];
-        console.log("Install", wheelUrls);
-        await micropip.install.callKwargs(wheelUrls, { keep_going: true });
-    }
-    console.log(`Install the requirements ${JSON.stringify(options.requirements)}`);
-    const micropip = pyodide.pyimport("micropip");
-    await micropip.install.callKwargs(options.requirements, { keep_going: true });
-    console.log("Archive the site-packages director(y|ies)");
-    const archiveFilePath = "/tmp/site-packages-snapshot.tar.gz";
-    await pyodide.runPythonAsync(`
+var D=Object.create;var m=Object.defineProperty;var T=Object.getOwnPropertyDescriptor;var S=Object.getOwnPropertyNames;var O=Object.getPrototypeOf,q=Object.prototype.hasOwnProperty;var f=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports);var P=(e,t,r,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of S(t))!q.call(e,o)&&o!==r&&m(e,o,{get:()=>t[o],enumerable:!(i=T(t,o))||i.enumerable});return e};var s=(e,t,r)=>(r=e!=null?D(O(e)):{},P(t||!e||!e.__esModule?m(r,"default",{value:e,enumerable:!0}):r,e));var h=f(d=>{"use strict";Object.defineProperty(d,"__esModule",{value:!0});d.parseRequirementsTxt=void 0;function $(e){return e.split(`
+`).filter(t=>!t.startsWith("#")).map(t=>t.trim()).filter(t=>t!=="")}d.parseRequirementsTxt=$});var g=f(c=>{"use strict";var j=c&&c.__createBinding||(Object.create?function(e,t,r,i){i===void 0&&(i=r);var o=Object.getOwnPropertyDescriptor(t,r);(!o||("get"in o?!t.__esModule:o.writable||o.configurable))&&(o={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,i,o)}:function(e,t,r,i){i===void 0&&(i=r),e[i]=t[r]}),x=c&&c.__exportStar||function(e,t){for(var r in e)r!=="default"&&!Object.prototype.hasOwnProperty.call(t,r)&&j(t,e,r)};Object.defineProperty(c,"__esModule",{value:!0});x(h(),c)});var w=s(require("yargs")),v=require("yargs/helpers"),a=s(require("path")),n=s(require("fs/promises")),y=s(require("fs-extra")),b=s(require("node-fetch")),_=require("pyodide"),k=s(g());global.fetch=b.default;async function C(e){console.info("Copy the build directory (the bare built app files) to this directory...");let t=a.default.resolve(__dirname,"../build");if(!(await n.default.stat(t)).isDirectory())throw new Error(`The source ${t} does not exist.`);if(t===e.copyTo){console.warn(`sourceDir == destDir (${t}). Are you in the development environment? Skip copying the directory.`);return}if(e.keepOld)try{await n.default.access(e.copyTo),console.info(`${e.copyTo} already exists. Use it and skip copying.`);return}catch{throw new Error(`${e.copyTo} does not exist even though the \`keepOld\` option is specified`)}console.log(`Copy ${t} to ${e.copyTo}`),await n.default.rm(e.copyTo,{recursive:!0,force:!0}),await y.default.copy(t,e.copyTo)}async function u(e,t){console.log(`Install the local wheel ${t}`);let r=await n.default.readFile(t),i="/tmp/"+a.default.basename(t);e.FS.writeFile(i,r);let o=e.pyimport("micropip"),l=`emfs:${i}`;console.log(`Install ${l}`),await o.install.callKwargs(l,{keep_going:!0})}async function K(e){console.info("Create the site-packages snapshot file...");let t=await(0,_.loadPyodide)();if(await t.loadPackage(["micropip"]),e.useLocalKernelWheels){let l=a.default.dirname(require.resolve("@stlite/kernel")),p=a.default.resolve(l,"../py");await u(t,a.default.join(p,"tornado/dist/tornado-6.2-py3-none-any.whl")),await u(t,a.default.join(p,"stlite-pyarrow/dist/stlite_pyarrow-0.1.0-py3-none-any.whl")),await u(t,a.default.join(p,"streamlit/lib/dist/streamlit-1.13.0-py2.py3-none-any.whl"))}else{let l=t.pyimport("micropip"),p=["https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/tornado/dist/tornado-6.2-py3-none-any.whl","https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/stlite-pyarrow/dist/stlite_pyarrow-0.1.0-py3-none-any.whl","https://cdn.jsdelivr.net/npm/@stlite/kernel@0.10.1/py/streamlit/lib/dist/streamlit-1.13.0-py2.py3-none-any.whl"];console.log("Install",p),await l.install.callKwargs(p,{keep_going:!0})}console.log(`Install the requirements ${JSON.stringify(e.requirements)}`),await t.pyimport("micropip").install.callKwargs(e.requirements,{keep_going:!0}),console.log("Archive the site-packages director(y|ies)");let i="/tmp/site-packages-snapshot.tar.gz";await t.runPythonAsync(`
     import tarfile
     import site
 
     site_packages_dirs = site.getsitepackages()
 
-    tar_file_name = '${archiveFilePath}'
+    tar_file_name = '${i}'
     with tarfile.open(tar_file_name, mode='w:gz') as gzf:
         for site_packages in site_packages_dirs:
             gzf.add(site_packages)
-  `);
-    console.log("Extract the archive file from EMFS");
-    const archiveBin = pyodide.FS.readFile(archiveFilePath);
-    console.log(`Save the archive file (${options.saveTo})`);
-    await promises_1.default.writeFile(options.saveTo, archiveBin);
-}
-async function copyStreamlitAppDirectory(options) {
-    console.info("Copy the Streamlit app directory...");
-    console.log(`Copy ${options.sourceDir} to ${options.copyTo}`);
-    await promises_1.default.rm(options.copyTo, { recursive: true, force: true });
-    await fs_extra_1.default.copy(options.sourceDir, options.copyTo);
-}
-// Original: packages/sharing-editor/bin/gen-sample-app-manifests-json.ts
-// TODO: Be DRY
-async function readRequirements(requirementsTxtPath) {
-    const requirementsTxtData = await promises_1.default.readFile(requirementsTxtPath, {
-        encoding: "utf-8",
-    });
-    return requirementsTxtData
-        .split("\n")
-        .map((r) => r.trim())
-        .filter((r) => r !== "");
-}
-// Original: kernel/src/requirements.ts
-// TODO: Be DRY
-function verifyRequirements(requirements) {
-    requirements.forEach((req) => {
-        let url;
-        try {
-            url = new URL(req);
-        }
-        catch {
-            // `req` is not a URL -> OK
-            return;
-        }
-        // Ref: The scheme checker in the micropip implementation is https://github.com/pyodide/micropip/blob/v0.1.0/micropip/_compat_in_pyodide.py#L23-L26
-        if (url.protocol === "emfs:" || url.protocol === "file:") {
-            throw new Error(`"emfs:" and "file:" protocols are not allowed for the requirement (${req})`);
-        }
-    });
-}
-(0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
-    .command("* <appHomeDirSource> [packages..]", "Put the user code and data and the snapshot of the required packages into the build artifact.", () => { }, (argv) => {
-    console.info(argv);
-})
-    .positional("appHomeDirSource", {
-    describe: "The source directory of the user code and data that will be mounted in the Pyodide file system at app runtime",
-    type: "string",
-    demandOption: true,
-})
-    .positional("packages", {
-    describe: "Package names to install.",
-    type: "string",
-    array: true,
-})
-    .options("requirement", {
-    describe: "Install from the given requirements file. This option can be used multiple times.",
-    array: true,
-    type: "string",
-    alias: "r",
-    default: [],
-})
-    .options("localKernelWheels", {
-    describe: "Use the locally installed kernel wheels",
-    type: "boolean",
-    alias: "l",
-    default: false,
-})
-    .options("keepOldBuild", {
-    type: "boolean",
-    default: false,
-    alias: "k",
-    describe: "Keep the existing build directory contents except appHomeDir.",
-})
-    .parseAsync()
-    .then(async (args) => {
-    const destDir = path_1.default.resolve(process.cwd(), "./build");
-    try {
-        await promises_1.default.access(args.appHomeDirSource);
-    }
-    catch {
-        throw new Error(`${args.appHomeDirSource} does not exist.`);
-    }
-    let requirements = args.packages;
-    for (const requirementTxtFilePath of args.requirement) {
-        requirements = requirements.concat(await readRequirements(requirementTxtFilePath));
-    }
-    verifyRequirements(requirements);
-    await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
-    await createSitePackagesSnapshot({
-        useLocalKernelWheels: args.localKernelWheels,
-        requirements: requirements,
-        saveTo: path_1.default.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
-    });
-    await copyStreamlitAppDirectory({
-        sourceDir: args.appHomeDirSource,
-        copyTo: path_1.default.resolve(destDir, "./streamlit_app"), // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
-    });
-});
+  `),console.log("Extract the archive file from EMFS");let o=t.FS.readFile(i);console.log(`Save the archive file (${e.saveTo})`),await n.default.writeFile(e.saveTo,o)}async function F(e){console.info("Copy the Streamlit app directory..."),console.log(`Copy ${e.sourceDir} to ${e.copyTo}`),await n.default.rm(e.copyTo,{recursive:!0,force:!0}),await y.default.copy(e.sourceDir,e.copyTo)}async function B(e){let t=await n.default.readFile(e,{encoding:"utf-8"});return(0,k.parseRequirementsTxt)(t)}function E(e){e.forEach(t=>{let r;try{r=new URL(t)}catch{return}if(r.protocol==="emfs:"||r.protocol==="file:")throw new Error(`"emfs:" and "file:" protocols are not allowed for the requirement (${t})`)})}(0,w.default)((0,v.hideBin)(process.argv)).command("* <appHomeDirSource> [packages..]","Put the user code and data and the snapshot of the required packages into the build artifact.",()=>{},e=>{console.info(e)}).positional("appHomeDirSource",{describe:"The source directory of the user code and data that will be mounted in the Pyodide file system at app runtime",type:"string",demandOption:!0}).positional("packages",{describe:"Package names to install.",type:"string",array:!0}).options("requirement",{describe:"Install from the given requirements file. This option can be used multiple times.",array:!0,type:"string",alias:"r",default:[]}).options("localKernelWheels",{describe:"Use the locally installed kernel wheels",type:"boolean",alias:"l",default:!1}).options("keepOldBuild",{type:"boolean",default:!1,alias:"k",describe:"Keep the existing build directory contents except appHomeDir."}).parseAsync().then(async e=>{let t=a.default.resolve(process.cwd(),"./build");try{await n.default.access(e.appHomeDirSource)}catch{throw new Error(`${e.appHomeDirSource} does not exist.`)}let r=e.packages;for(let i of e.requirement)r=r.concat(await B(i));E(r),await C({copyTo:t,keepOld:e.keepOldBuild}),await K({useLocalKernelWheels:e.localKernelWheels,requirements:r,saveTo:a.default.resolve(t,"./site-packages-snapshot.tar.gz")}),await F({sourceDir:e.appHomeDirSource,copyTo:a.default.resolve(t,"./streamlit_app")})});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stlite/desktop",
-  "version": "0.18.2",
+  "version": "0.19.0",
   "license": "Apache-2.0",
   "homepage": "/",
   "main": "./build/electron/main.js",
@@ -20,11 +20,12 @@
     "start:electron": "tsc -p electron -w",
     "build:electron": "tsc -p electron",
     "build:pyodide": "curl -L https://github.com/pyodide/pyodide/releases/download/0.21.3/pyodide-build-0.21.3.tar.bz2 | tar xj -C ./build --files-from=./pyodide-files.txt",
-    "build:bin": "tsc -p bin && sed -i'' -e '1 s/^#!.*$/#!\\/usr\\/bin\\/env node/' ./bin/*.js",
+    "build:bin": "esbuild ./bin/dump_artifacts.ts --bundle --minify --platform=node --external:@stlite/kernel --external:node-fetch --external:pyodide --external:yargs --external:fs-extra --outfile=./bin/dump_artifacts.js && sed -i'' -e '1 s/^#!.*$/#!\\/usr\\/bin\\/env node/' ./bin/*.js",
     "typecheck": "yarn tsc --noEmit -p electron",
     "start": "concurrently \"cross-env BROWSER=none yarn start:web\" \"wait-on http://localhost:3000 && yarn start:electron\" \"wait-on http://localhost:3000 && tsc -p electron && cross-env NODE_ENV=\"development\" electron .\"",
     "build:app": "yarn build:web && yarn build:electron && yarn build:pyodide",
     "build": "yarn build:app && yarn build:bin",
+    "dump": "dump-stlite-desktop-artifacts",
     "serve": "cross-env NODE_ENV=production electron .",
     "pack": "electron-builder --dir",
     "dist": "electron-builder",
@@ -49,7 +50,8 @@
   },
   "devDependencies": {
     "@craco/craco": "^6.1.2",
-    "@stlite/kernel": "^0.18.2",
+    "@stlite/common": "^0.19.0",
+    "@stlite/kernel": "^0.19.0",
     "@testing-library/react": "^11.2.7",
     "@testing-library/user-event": "^13.1.9",
     "@types/jest": "^26.0.19",
@@ -60,6 +62,7 @@
     "electron": "20.2.0",
     "electron-builder": "^23.3.3",
     "electron-reload": "^2.0.0-alpha.1",
+    "esbuild": "^0.16.6",
     "fs-extra": "^10.1.0",
     "node-fetch": "2",
     "pyodide": "^0.21.3",