@lakutata/builder 3.0.0-beta.0

package/dist/esm/lib/Native.js

@@ -0,0 +1,533 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
+import { createRequire, isBuiltin } from 'node:module';
+import path from 'node:path';
+import { gzipSync } from 'node:zlib';
+/**
+ * The shared libraries: the addons, and the libraries they are linked with (sharp and libvips)
+ */
+const ADDON = /\.node$/;
+const LIBRARY = /\.(node|dylib|dll|so)$|\.so\.\d+(\.\d+)*$/;
+const RESOLVING = 'lakutataNativeResolving';
+/**
+ * The files of the packages node never loads: their types, their source maps, their sources in TypeScript, their
+ * documentation
+ */
+const UNUSED = /\.(d\.[cm]?ts|[cm]?ts|map|md|markdown)$/i;
+/**
+ * The JavaScript files, which a package keeps when they are loaded (traced)
+ */
+const SCRIPT = /\.[cm]?js$/;
+/**
+ * A require esbuild cannot follow loading a file of its package by its path (require(path.join(__dirname, name)))
+ */
+const OWN_FILE = /__dirname|__filename|import\.meta|['"`]\.\.?\//;
+/**
+ * The require of an ES module, createRequire(import.meta.url): in the bundle, it is the require of the bundle
+ */
+const MODULE_REQUIRE = /\b(?:[A-Za-z_$][\w$]*\.)?createRequire\s*\(\s*import\.meta\.url\s*\)/g;
+/**
+ * The name of the package of a module specifier ("@scope/name/sub/path" → "@scope/name"), undefined for a path
+ * @param specifier
+ */
+export function specifierPackage(specifier) {
+    if (specifier.startsWith('.') || path.isAbsolute(specifier))
+        return undefined;
+    const parts = specifier.split('/');
+    return specifier.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0];
+}
+/**
+ * The files of a package directory, without the packages it holds (node_modules)
+ * @param directory
+ */
+function packageFiles(directory) {
+    const files = [];
+    const walk = (current) => {
+        for (const entry of readdirSync(current, { withFileTypes: true })) {
+            if (entry.name === 'node_modules')
+                continue;
+            const file = path.join(current, entry.name);
+            let stats;
+            try {
+                stats = statSync(file);
+            }
+            catch {
+                //A broken link
+                continue;
+            }
+            if (stats.isDirectory())
+                walk(file);
+            else if (stats.isFile())
+                files.push(file);
+        }
+    };
+    walk(directory);
+    return files;
+}
+/**
+ * The directories node looks a package up in from a directory of the tree, the nearest first
+ * @param directory
+ * @param name
+ */
+function lookupDirectories(directory, name) {
+    const parts = directory.split('/');
+    const lookups = [];
+    for (let index = parts.length; index >= 0; index--) {
+        const prefix = parts.slice(0, index);
+        if (prefix.at(-1) === 'node_modules')
+            continue;
+        lookups.push([...prefix, 'node_modules', name].join('/'));
+    }
+    return lookups;
+}
+/**
+ * Find the native addons of an application while it is bundled: the packages holding addons (and the packages loading
+ * them with computed paths, such as sharp) are left out of the bundle and gathered, with the packages they load, into
+ * a native tree; the addons of the application itself are loaded from the tree
+ */
+export class NativeCollector {
+    /**
+     * @param project the directory of the project, whose files are bundled
+     * @param runtime the runtime loading the tree: Bun resolves the packages with its condition (bun) first
+     */
+    constructor(project, runtime = 'node') {
+        this.project = project;
+        this.runtime = runtime;
+        this.packages = new Map();
+        this.roots = new Map();
+        /**
+         * The packages left out of the bundle, by the name the application loads them with, and the files it loads
+         */
+        this.externals = new Map();
+        /**
+         * The addons of the application itself, by their key in the tree
+         */
+        this.addons = new Map();
+        /**
+         * The files of the packages of the tree which are loaded, by package (traced)
+         */
+        this.loaded = new Map();
+        /**
+         * The packages of the tree kept as a whole: the ones which are not traced, the ones loading their files by paths
+         * esbuild cannot follow
+         */
+        this.whole = new Set();
+        /**
+         * The packages each package of the tree requires, by their names (traced)
+         */
+        this.required = new Map();
+        this.project = realpathSync(project);
+    }
+    /**
+     * The directory of the package a file belongs to: the nearest directory with a named package.json
+     * @param file
+     */
+    packageRoot(file) {
+        let directory = path.dirname(file);
+        const visited = [];
+        let found;
+        while (true) {
+            if (this.roots.has(directory)) {
+                found = this.roots.get(directory);
+                break;
+            }
+            visited.push(directory);
+            const manifest = path.join(directory, 'package.json');
+            if (existsSync(manifest)) {
+                try {
+                    if (JSON.parse(readFileSync(manifest, 'utf8')).name) {
+                        found = directory;
+                        break;
+                    }
+                }
+                catch {
+                    //Not a package.json of a package
+                }
+            }
+            const parent = path.dirname(directory);
+            if (parent === directory)
+                break;
+            directory = parent;
+        }
+        for (const entry of visited)
+            this.roots.set(entry, found);
+        return found;
+    }
+    /**
+     * A package: its manifest and whether it holds addons or shared libraries
+     * @param root
+     */
+    package(root) {
+        if (!this.packages.has(root)) {
+            let info;
+            try {
+                const manifest = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8'));
+                const files = packageFiles(root);
+                info = {
+                    root: root,
+                    name: manifest.name,
+                    manifest: manifest,
+                    addons: files.some((file) => ADDON.test(file)),
+                    libraries: files.some((file) => LIBRARY.test(file))
+                };
+            }
+            catch {
+                info = undefined;
+            }
+            this.packages.set(root, info);
+        }
+        return this.packages.get(root);
+    }
+    /**
+     * The dependencies of a package, as node finds them from it: the path looked up (a link with pnpm) and the package
+     * @param root
+     */
+    dependencies(root) {
+        const manifest = this.package(root)?.manifest ?? {};
+        const names = new Set(Object.keys({ ...manifest.dependencies, ...manifest.optionalDependencies, ...manifest.peerDependencies }));
+        const found = [];
+        for (const name of names) {
+            let directory = root;
+            while (true) {
+                const lookup = path.join(directory, 'node_modules', name);
+                if (existsSync(path.join(lookup, 'package.json'))) {
+                    found.push({ name: name, lookup: lookup, root: realpathSync(lookup) });
+                    break;
+                }
+                const parent = path.dirname(directory);
+                if (parent === directory)
+                    break;
+                directory = parent;
+            }
+        }
+        return found;
+    }
+    /**
+     * Whether a package is left out of the bundle: it holds addons, or one of its dependencies does (it loads them,
+     * with computed paths often), and it is not the project
+     * @param root
+     */
+    isNative(root) {
+        if (!root || root === this.project)
+            return false;
+        const info = this.package(root);
+        if (!info)
+            return false;
+        return info.addons || this.dependencies(root).some(({ root: dependency }) => !!this.package(dependency)?.addons);
+    }
+    /**
+     * Resolve an import as esbuild does
+     * @param build
+     * @param args
+     */
+    async resolve(build, args) {
+        const result = await build.resolve(args.path, { kind: args.kind, resolveDir: args.resolveDir, importer: args.importer, pluginData: { [RESOLVING]: true } });
+        return result.errors.length || result.external || !result.path ? undefined : realpathSync(result.path);
+    }
+    /**
+     * The plugin of the bundle: the native packages are external, the addons of the application are loaded from the tree
+     */
+    plugin() {
+        return {
+            name: 'lakutata-native',
+            setup: (build) => {
+                build.onResolve({ filter: /.*/ }, async (args) => {
+                    if (args.pluginData?.[RESOLVING] || isBuiltin(args.path) || args.kind === 'entry-point')
+                        return undefined;
+                    const file = await this.resolve(build, args);
+                    if (!file)
+                        return undefined;
+                    const root = this.packageRoot(file);
+                    if (this.isNative(root)) {
+                        const name = specifierPackage(args.path);
+                        if (!name)
+                            throw new Error(`${args.importer} loads ${args.path} of the native package ${this.package(root).name} by its path: load it by its name`);
+                        const external = this.externals.get(name) ?? { root: root, entries: new Set() };
+                        external.entries.add(file);
+                        //The bundle is CommonJS: the require of the run loads the file of the require condition
+                        if (args.kind !== 'require-call') {
+                            const required = await this.resolve(build, { ...args, kind: 'require-call' });
+                            if (required)
+                                external.entries.add(required);
+                        }
+                        this.externals.set(name, external);
+                        return { path: args.path, external: true };
+                    }
+                    if (ADDON.test(file)) {
+                        const key = `.lakutata/addons/${createHash('sha256').update(file).digest('hex').slice(0, 16)}-${path.basename(file)}`;
+                        this.addons.set(key, file);
+                        return { path: key, namespace: 'lakutata-native' };
+                    }
+                    return undefined;
+                });
+                //The requires of the ES modules are the one of the bundle: esbuild follows what they load (the addons)
+                build.onLoad({ filter: /\.(c|m)?js$/ }, async (args) => {
+                    const source = await readFile(args.path, 'utf8');
+                    if (!MODULE_REQUIRE.test(source))
+                        return undefined;
+                    MODULE_REQUIRE.lastIndex = 0;
+                    return { contents: source.replace(MODULE_REQUIRE, 'require'), loader: 'js', resolveDir: path.dirname(args.path) };
+                });
+                //The addons of the application are loaded by the require of the bundle, from the extracted tree
+                build.onLoad({ filter: /.*/, namespace: 'lakutata-native' }, (args) => ({
+                    contents: `module.exports = require.lakutataNative(${JSON.stringify(args.path)})`,
+                    loader: 'js'
+                }));
+            }
+        };
+    }
+    /**
+     * The packages a native package loads at run time: the JavaScript it requires (followed by esbuild), the packages of
+     * shared libraries it depends on (the libraries linked with its addons), and the native packages it loads
+     * @param build the build function of esbuild
+     * @param root
+     * @param entries the files of the package which are loaded
+     * @param included the packages of the tree
+     */
+    async trace(build, root, entries, included) {
+        const pending = [{ root: root, entries: entries }];
+        const traced = new Set();
+        while (pending.length) {
+            const current = pending.shift();
+            const key = `${current.root}\0${[...current.entries].sort().join('\0')}`;
+            if (traced.has(key))
+                continue;
+            traced.add(key);
+            included.add(current.root);
+            //A package esbuild cannot trace (a glob of its requires matching nothing) is kept as a whole
+            const result = await build({
+                entryPoints: [...current.entries],
+                bundle: true,
+                write: false,
+                //Not written: a package loaded by several of its files is several entries, which need a directory
+                outdir: path.join(current.root, '.lakutata-trace'),
+                metafile: true,
+                platform: 'node',
+                format: 'cjs',
+                conditions: this.runtime === 'bun' ? ['bun'] : [],
+                logLevel: 'silent',
+                //The addons are loaded at run time (the paths of their globs may match nothing: sharp built from its sources)
+                external: ['*.node'],
+                //Reported in the node_modules too: the requires it cannot follow
+                logOverride: { 'unsupported-require-call': 'warning' },
+                plugins: [{
+                        name: 'lakutata-native-trace',
+                        setup: (trace) => {
+                            trace.onResolve({ filter: /.*/ }, async (args) => {
+                                if (args.pluginData?.[RESOLVING] || isBuiltin(args.path) || args.kind === 'entry-point')
+                                    return undefined;
+                                const file = await this.resolve(trace, args);
+                                //The requires which cannot be followed (computed paths) are left to the run time
+                                if (!file || ADDON.test(file))
+                                    return { path: args.path, external: true };
+                                const dependency = this.packageRoot(file);
+                                //The package it requires by this name (the ones it does not declare too)
+                                const name = specifierPackage(args.path);
+                                const importer = args.importer ? this.packageRoot(realpathSync(args.importer)) : undefined;
+                                if (name && importer && dependency && importer !== dependency)
+                                    this.requires(importer).set(name, dependency);
+                                if (dependency !== current.root && this.isNative(dependency)) {
+                                    pending.push({ root: dependency, entries: new Set([file]) });
+                                    return { path: args.path, external: true };
+                                }
+                                return undefined;
+                            });
+                        }
+                    }]
+            }).catch(() => undefined);
+            if (!result)
+                this.whole.add(current.root);
+            for (const input of Object.keys(result?.metafile.inputs ?? {})) {
+                const file = realpathSync(path.resolve(input));
+                const dependency = this.packageRoot(file);
+                if (!dependency || dependency === this.project)
+                    continue;
+                included.add(dependency);
+                const files = this.loaded.get(dependency) ?? new Set();
+                this.loaded.set(dependency, files.add(file));
+            }
+            //The packages loading their files by paths esbuild cannot follow are kept as a whole
+            for (const warning of result?.warnings ?? []) {
+                if (!warning.location || !OWN_FILE.test(warning.location.lineText.slice(warning.location.column, warning.location.column + 200)))
+                    continue;
+                const dependency = this.packageRoot(realpathSync(path.resolve(warning.location.file)));
+                if (dependency)
+                    this.whole.add(dependency);
+            }
+            //The packages it declares which its code names: the ones it loads by requires esbuild does not follow
+            //(createRequire(import.meta.url)('node-gyp-build')), not the ones installing it (node-gyp, prebuild-install)
+            const sources = (result ? Object.keys(result.metafile.inputs).map((input) => realpathSync(path.resolve(input))) : packageFiles(current.root))
+                .filter((file) => this.packageRoot(file) === current.root && /\.[cm]?[jt]s$/.test(file))
+                .map((file) => readFileSync(file, 'utf8'))
+                .join('\n');
+            for (const dependency of this.dependencies(current.root)) {
+                if (dependency.root === this.project || included.has(dependency.root))
+                    continue;
+                if (!new RegExp(`(['"\`])${dependency.name.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')}\\1`).test(sources))
+                    continue;
+                let entry;
+                try {
+                    entry = createRequire(path.join(current.root, 'package.json')).resolve(dependency.name);
+                }
+                catch {
+                    //A package without a CommonJS entry: kept as a whole
+                }
+                if (entry && !isBuiltin(entry))
+                    pending.push({ root: dependency.root, entries: new Set([realpathSync(entry)]) });
+                else {
+                    included.add(dependency.root);
+                    this.whole.add(dependency.root);
+                }
+            }
+        }
+        //The packages of the shared libraries (the addons of the platform packages, the libraries they are linked with)
+        const libraries = [...included];
+        while (libraries.length) {
+            for (const { root: dependency } of this.dependencies(libraries.pop())) {
+                if (!included.has(dependency) && this.package(dependency)?.libraries) {
+                    included.add(dependency);
+                    libraries.push(dependency);
+                }
+            }
+        }
+    }
+    /**
+     * The packages a package requires, by their names
+     * @param root
+     */
+    requires(root) {
+        const required = this.required.get(root) ?? new Map();
+        this.required.set(root, required);
+        return required;
+    }
+    /**
+     * The layout of the packages of the tree, without links (as npm installs them, the tree is moved, copied, archived
+     * and installed as it is): the packages the application loads in node_modules, the packages each package requires
+     * (traced) or declares where node finds them from it: shared when node finds the same package, at the top when its
+     * name is free there, else in the node_modules of the package
+     * @param included the packages of the tree
+     * @return the packages by their directories in the tree
+     */
+    layout(included) {
+        const placed = new Map();
+        const queue = [];
+        const place = (directory, root) => queue.push(placed.set(directory, root) && directory);
+        const drain = () => {
+            while (queue.length) {
+                const directory = queue.shift();
+                const root = placed.get(directory);
+                const needed = new Map(this.required.get(root));
+                for (const dependency of this.dependencies(root)) {
+                    if (included.has(dependency.root) && !needed.has(dependency.name))
+                        needed.set(dependency.name, dependency.root);
+                }
+                for (const [name, dependency] of [...needed].sort(([a], [b]) => a.localeCompare(b))) {
+                    if (dependency === root || !included.has(dependency))
+                        continue;
+                    const found = lookupDirectories(directory, name).find((lookup) => placed.has(lookup));
+                    if (found !== undefined && placed.get(found) === dependency)
+                        continue;
+                    place(found === undefined ? `node_modules/${name}` : `${directory}/node_modules/${name}`, dependency);
+                }
+            }
+        };
+        for (const [name, { root }] of [...this.externals].sort(([a], [b]) => a.localeCompare(b)))
+            place(`node_modules/${name}`, root);
+        drain();
+        //The packages reached otherwise (by the libraries of a package): at the top
+        const reached = new Set(placed.values());
+        for (const root of [...included].sort()) {
+            const name = this.package(root)?.name;
+            if (reached.has(root) || !name || placed.has(`node_modules/${name}`))
+                continue;
+            place(`node_modules/${name}`, root);
+        }
+        drain();
+        return placed;
+    }
+    /**
+     * The files of a package of the tree which are loaded at run time, without the ones node never loads (UNUSED): the
+     * packages of shared libraries (which find their addons by computed paths), and the ones kept as a whole, have all
+     * their files; the others have the JavaScript files they load (traced) and all their other files (the data they read)
+     * @param root
+     */
+    packageFiles(root) {
+        const files = packageFiles(root).filter((file) => !UNUSED.test(file));
+        const loaded = this.loaded.get(root);
+        if (!loaded || this.whole.has(root) || this.package(root)?.libraries)
+            return files;
+        return files.filter((file) => !SCRIPT.test(file) || loaded.has(file));
+    }
+    /**
+     * The native tree of the application, once it is bundled; undefined when it has no native addon
+     */
+    async collect() {
+        if (!this.externals.size && !this.addons.size)
+            return undefined;
+        const { build } = await import('esbuild');
+        const included = new Set();
+        for (const { root, entries } of this.externals.values())
+            await this.trace(build, root, entries, included);
+        const files = [];
+        for (const [directory, root] of [...this.layout(included)].sort(([a], [b]) => a.localeCompare(b))) {
+            for (const file of this.packageFiles(root))
+                files.push({ path: `${directory}/${path.relative(root, file).split(path.sep).join('/')}`, source: file, mode: statSync(file).mode & 0o777 });
+        }
+        for (const [key, file] of this.addons)
+            files.push({ path: key, source: file, mode: statSync(file).mode & 0o777 });
+        return { modules: [...this.externals.keys()].sort(), files: files, links: [], hash: hashNativeTree(files, []) };
+    }
+}
+/**
+ * The hash of a native tree, which names the directory it is extracted into: its files and their content, its links
+ * @param files
+ * @param links
+ */
+export function hashNativeTree(files, links) {
+    const hash = createHash('sha256');
+    for (const file of files)
+        hash.update(`${file.path}\0${file.mode}\0`).update(readFileSync(file.source));
+    for (const link of links)
+        hash.update(`${link.path}\0${link.target}\0`);
+    return hash.digest('hex').slice(0, 16);
+}
+/**
+ * Write a native tree into a directory (next to a bundle)
+ * @param tree
+ * @param directory
+ */
+export async function writeNativeTree(tree, directory) {
+    await rm(directory, { recursive: true, force: true });
+    for (const file of tree.files) {
+        const target = path.join(directory, ...file.path.split('/'));
+        await mkdir(path.dirname(target), { recursive: true });
+        await writeFile(target, readFileSync(file.source), { mode: file.mode });
+    }
+    for (const link of tree.links) {
+        const at = path.join(directory, ...link.path.split('/'));
+        const target = path.join(directory, ...link.target.split('/'));
+        await mkdir(path.dirname(at), { recursive: true });
+        //The junctions of Windows need no privilege
+        if (process.platform === 'win32')
+            await symlink(target, at, 'junction');
+        else
+            await symlink(path.relative(path.dirname(at), target), at, 'dir');
+    }
+}
+/**
+ * Pack a native tree into the assets of an executable: its gzipped files and their index
+ * @param tree
+ */
+export function packNativeTree(tree) {
+    const chunks = [];
+    const files = [];
+    let offset = 0;
+    for (const file of tree.files) {
+        const content = readFileSync(file.source);
+        files.push({ path: file.path, offset: offset, size: content.length, mode: file.mode });
+        chunks.push(content);
+        offset += content.length;
+    }
+    return { index: { hash: tree.hash, files: files, links: tree.links }, data: gzipSync(Buffer.concat(chunks)) };
+}