@lakutata/builder 3.0.0-beta.0

package/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2023-present Lakutata
+
+All rights reserved
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,27 @@
+# @lakutata/builder
+
+The builder of the applications of the lakutata framework, used by the `bundle` and `executable` commands of
+`@lakutata/cli`, and usable from the scripts of a project:
+
+```js
+import {buildExecutable, buildInstaller, bundleProject} from '@lakutata/builder'
+
+//dist/<name>.cjs, run by node, its native addons and its resources next to it
+await bundleProject({project: process.cwd()})
+//dist/<name>, which runs without Node.js: Node.js (by default) or Bun embedded, one file or a directory
+await buildExecutable({project: process.cwd(), runtime: 'node', layout: 'directory'})
+//dist/installer: the installers of the platform (deb, rpm, apk, archlinux, tar.gz on Linux; pkg, tar.gz on macOS;
+//nsis, msi, zip on Windows), its service registered
+await buildInstaller({project: process.cwd()})
+```
+
+The application is compiled first (`tsc`, which emits the metadata of the decorators). The bundle is one CommonJS file
+(esbuild); the native addons, which cannot be bundled, are gathered with what they load at run time into a native tree;
+the resources of the application (the `lakutata.resources` of its `package.json`) are found at run time by
+`resourcesPath()` of lakutata. The installers are configured by the `lakutata.installer` of the `package.json`. See the
+documentation of `lakutata package bundle`, `lakutata package exe` and `lakutata package installer` in `@lakutata/cli`.
+
+The installers of Linux are tested in containers (Docker): `npm run test:installer` makes them on Debian and Alpine,
+installs, upgrades and removes them on Debian and Fedora (systemd) and Alpine (OpenRC). The ones of macOS are installed
+in a virtual machine of macOS (`npm run test:installer:mac`, tart), the ones of Windows on Windows as an administrator
+(`LAKUTATA_WINDOWS_INSTALL=1 npm run test:installer:windows`).

package/dist/esm/index.js

@@ -0,0 +1,13 @@
+/**
+ * The builder of the applications of lakutata: their bundles (one CommonJS file run by node) and their executables (the
+ * single executable applications of Node.js, or Bun), with their native addons and their resources
+ */
+export { bundleProject, programName, resourceFiles } from './lib/Bundle.js';
+export { buildExecutable } from './lib/Executable.js';
+export { NativeCollector, packNativeTree, writeNativeTree } from './lib/Native.js';
+export { buildInstaller } from './lib/Installer.js';
+export { LINUX_TARGETS } from './lib/linux/Linux.js';
+export { INSTALLER_HOOKS, NATIVE_HOOKS } from './lib/Scripts.js';
+export { detectLibc } from './lib/Platform.js';
+export { WINDOWS_TARGETS } from './lib/win/Windows.js';
+export { MAC_TARGETS } from './lib/mac/Mac.js';

package/dist/esm/lib/Archive.js

@@ -0,0 +1,21 @@
+import { cp, rm, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { runCommand } from './Processes.js';
+/**
+ * The tar.gz archive of the directory of the executable (its native addons and its resources next to it), named after
+ * the package, its version, its platform and its architecture
+ * @param plan
+ * @param arch
+ * @param work
+ * @param outdir
+ */
+export async function archiveDirectory(plan, arch, work, outdir) {
+    const base = `${plan.name}-${plan.version}-${process.platform}-${arch}`;
+    await rm(path.join(work, base), { recursive: true, force: true });
+    await cp(plan.directory, path.join(work, base), { recursive: true, verbatimSymlinks: true });
+    const file = path.join(outdir, `${base}.tar.gz`);
+    const archived = await runCommand('tar', ['-czf', file, base], work);
+    if (archived.code !== 0)
+        throw new Error(`The tar.gz archive cannot be made: ${archived.output.trim()}`);
+    return { target: 'tar.gz', file: file, size: (await stat(file)).size };
+}

package/dist/esm/lib/Bundle.js

@@ -0,0 +1,162 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { chmod, cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { hashNativeTree, NativeCollector, writeNativeTree } from './Native.js';
+import { EMBEDDED_RESOURCES, nativeBanner, packagedBanner } from './NativeRuntime.js';
+/**
+ * The name of the program of a package: its first bin, or the name of the package without its scope
+ * @param manifest
+ */
+export function programName(manifest) {
+    if (manifest.bin && typeof manifest.bin === 'object') {
+        const [name] = Object.keys(manifest.bin);
+        if (name)
+            return name;
+    }
+    return String(manifest.name ?? 'app').replace(/^@[^/]+\//, '');
+}
+/**
+ * The ESM sources use import.meta.url, which the CommonJS bundle gives from its file
+ */
+const IMPORT_META_URL = '__lakutataImportMetaUrl';
+/**
+ * The first code of a bundle embedded in an executable of Bun, which bundles it again into a function without the
+ * arguments of the CommonJS modules: its file is the executable (as in the executables of Node.js), its require loads
+ * the modules of node and the ones of a node_modules next to the executable (the ones left out of the bundle)
+ */
+const BUN_PRELUDE = 'var __filename=process.execPath,__dirname=process.getBuiltinModule(\'node:path\').dirname(__filename),require=typeof require===\'function\'?require:process.getBuiltinModule(\'node:module\').createRequire(__filename);';
+/**
+ * The files of the resources of an application, declared by the "lakutata.resources" of its package.json: their paths in
+ * the resources ("/" separated) and their sources
+ * @param project
+ * @param manifest
+ */
+export function resourceFiles(project, manifest) {
+    const declarations = manifest.lakutata?.resources;
+    if (declarations === undefined)
+        return { declared: [], files: [] };
+    const valid = (declaration) => typeof declaration?.from === 'string' && typeof declaration?.to === 'string' &&
+        !!declaration.to && !path.isAbsolute(declaration.to) && !declaration.to.split(/[\\/]/).includes('..');
+    if (!Array.isArray(declarations) || !declarations.every(valid))
+        throw new Error('The resources of the application (lakutata.resources of its package.json) must be [{"from": "<file or directory of the project>", "to": "<its path in the resources>"}]');
+    const files = [];
+    for (const { from, to } of declarations) {
+        const source = path.resolve(project, from);
+        if (!existsSync(source))
+            throw new Error(`The resource ${from} does not exist: build it first`);
+        const target = to.split(/[\\/]/).filter(Boolean).join('/');
+        const walk = (file, at) => {
+            const stats = statSync(file);
+            if (stats.isDirectory())
+                for (const entry of readdirSync(file))
+                    walk(path.join(file, entry), `${at}/${entry}`);
+            else if (stats.isFile())
+                files.push({ path: at, source: file, mode: stats.mode & 0o777 });
+        };
+        walk(source, target);
+    }
+    return { declared: declarations.map(({ to }) => to), files: files };
+}
+/**
+ * The code with a code before it, after its hashbang
+ * @param code
+ * @param first
+ */
+function prepend(code, first) {
+    const hashbang = /^#![^\n]*\n/.exec(code);
+    return hashbang ? `${hashbang[0]}${first}${code.slice(hashbang[0].length)}` : `${first}${code}`;
+}
+/**
+ * Bundle the compiled application and its dependencies into one CommonJS file, run by node and embedded in the executables.
+ * The sources are compiled by tsc first: the metadata of the decorators (the injections, the DTOs) are emitted by it.
+ * The native addons cannot be bundled: their packages (and what they load) are gathered into a native tree, which the
+ * require of the bundle loads them from
+ * @param options
+ */
+export async function bundleProject(options) {
+    const project = path.resolve(options.project);
+    const manifestFile = path.join(project, 'package.json');
+    if (!existsSync(manifestFile))
+        throw new Error(`${project} has no package.json`);
+    const manifest = JSON.parse(await readFile(manifestFile, 'utf8'));
+    const name = programName(manifest);
+    const entry = path.resolve(project, options.entry ?? manifest.main ?? 'build/App.js');
+    if (!existsSync(entry))
+        throw new Error(`The entry ${path.relative(process.cwd(), entry) || entry} does not exist: build the project first`);
+    const outfile = path.resolve(project, options.outfile ?? path.join('dist', `${name}.cjs`));
+    const collector = new NativeCollector(project, options.runtime);
+    //esbuild is loaded by the commands which need it
+    const { build } = await import('esbuild');
+    const result = await build({
+        entryPoints: [entry],
+        outfile: outfile,
+        write: false,
+        bundle: true,
+        platform: 'node',
+        target: `node${process.versions.node.split('.')[0]}`,
+        format: 'cjs',
+        minify: options.minify ?? true,
+        //The names of the classes are the ones of the objects of the container and of the errors
+        keepNames: true,
+        external: options.external ?? [],
+        //The dynamic imports of the modules left out are requires (the require of the bundle loads the native ones)
+        supported: { 'dynamic-import': false },
+        banner: { js: `var ${IMPORT_META_URL}=require('node:url').pathToFileURL(__filename).href;` },
+        define: {
+            'import.meta.url': IMPORT_META_URL,
+            //Bun replaces process.env.NODE_ENV with the mode of its build: the one of the run is read
+            ...options.runtime === 'bun' ? { 'process.env.NODE_ENV': 'globalThis.process.env.NODE_ENV' } : {}
+        },
+        plugins: [collector.plugin()],
+        logLevel: 'silent'
+    }).catch((error) => {
+        const messages = (error.errors ?? []).map(({ text, location }) => location ? `${location.file}:${location.line}: ${text}` : text);
+        //The bundles are CommonJS, which the executables of Node.js run
+        const hint = messages.some((message) => message.includes('Top-level await')) ? '\nThe bundles are CommonJS modules, without top-level await: await in an async function' : '';
+        throw new Error(`The bundle failed:\n  ${messages.length ? messages.join('\n  ') : error.message}${hint}`);
+    });
+    let code = result.outputFiles[0].text;
+    const embedded = options.native === 'embedded';
+    let native = await collector.collect();
+    const resources = resourceFiles(project, manifest);
+    const base = path.basename(outfile, path.extname(outfile));
+    const nativeDirectory = path.resolve(path.dirname(outfile), options.directories?.native ?? `${base}.native`);
+    const resourcesDirectory = path.resolve(path.dirname(outfile), options.directories?.resources ?? `${base}.resources`);
+    await rm(nativeDirectory, { recursive: true, force: true });
+    await rm(resourcesDirectory, { recursive: true, force: true });
+    //The resources of an executable are extracted with its native tree
+    if (embedded && resources.files.length) {
+        const files = [...native?.files ?? [], ...resources.files.map((file) => ({ ...file, path: `${EMBEDDED_RESOURCES}/${file.path}` }))];
+        native = { modules: native?.modules ?? [], files: files, links: native?.links ?? [], hash: hashNativeTree(files, native?.links ?? []) };
+    }
+    if (native) {
+        //The require loading the native packages is the first code, after the hashbang
+        code = prepend(code, nativeBanner({ name: name, hash: native.hash, directory: path.basename(nativeDirectory), modules: native.modules, embedded: embedded }));
+        if (!embedded)
+            await writeNativeTree(native, nativeDirectory);
+    }
+    if (!embedded) {
+        for (const file of resources.files) {
+            const target = path.join(resourcesDirectory, ...file.path.split('/'));
+            await mkdir(path.dirname(target), { recursive: true });
+            await cp(file.source, target);
+        }
+    }
+    code = prepend(code, packagedBanner(path.basename(resourcesDirectory)));
+    //Before the require of the native tree, which loads the modules of node by it
+    if (options.runtime === 'bun')
+        code = prepend(code, BUN_PRELUDE);
+    await mkdir(path.dirname(outfile), { recursive: true });
+    await writeFile(outfile, code);
+    await chmod(outfile, 0o755);
+    return {
+        name: name,
+        entry: entry,
+        outfile: outfile,
+        size: (await stat(outfile)).size,
+        native: native,
+        nativeDirectory: native && !embedded ? nativeDirectory : undefined,
+        resources: resources.declared,
+        resourcesDirectory: resources.files.length && !embedded ? resourcesDirectory : undefined
+    };
+}

package/dist/esm/lib/Executable.js

@@ -0,0 +1,221 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { chmod, copyFile, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import os from 'node:os';
+import path from 'node:path';
+import { detectLibc } from './Platform.js';
+import { runCommand } from './Processes.js';
+import { bundleProject, programName } from './Bundle.js';
+import { applicationIcon, icoFile, setExecutableIcon } from './Icons.js';
+import { packNativeTree } from './Native.js';
+import { BUN_NATIVE_ASSETS, NATIVE_DATA_ASSET, NATIVE_INDEX_ASSET } from './NativeRuntime.js';
+/**
+ * The fuse of the Node.js binaries telling that a single executable application is injected
+ */
+const SEA_FUSE = 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2';
+/**
+ * Run a command which must succeed
+ * @param command
+ * @param args
+ * @param cwd
+ */
+async function must(command, args, cwd) {
+    const result = await runCommand(command, args, cwd);
+    if (result.code !== 0)
+        throw new Error(`${command} ${args.join(' ')} failed: ${result.output.trim()}`);
+}
+/**
+ * Run a function without the output of the console: postject prints the warnings of LIEF, which it binds when it is called
+ * ("Can't find string offset for section name", the signature of node.exe changed by the injection)
+ * @param run
+ */
+async function quiet(run) {
+    const { log, warn } = console;
+    console.log = console.warn = () => undefined;
+    try {
+        return await run();
+    }
+    finally {
+        console.log = log;
+        console.warn = warn;
+    }
+}
+/**
+ * The bun of the project (the bun package of npm, in its node_modules or in the ones of its workspace), else the one of
+ * the system
+ * @param project
+ * @return the command, and the version of Bun
+ */
+async function findBun(project) {
+    const candidates = [];
+    try {
+        const root = path.dirname(createRequire(path.join(project, 'package.json')).resolve('bun/package.json'));
+        candidates.push(path.join(root, 'bin', 'bun.exe'));
+    }
+    catch {
+        //Not installed in the project
+    }
+    candidates.push('bun');
+    for (const command of candidates) {
+        if (command !== 'bun' && !existsSync(command))
+            continue;
+        const result = await runCommand(command, ['--version'], project).catch(() => undefined);
+        if (result?.code === 0)
+            return { command: command, version: result.output.trim() };
+    }
+    throw new Error('Bun is not installed: install it in the project (npm install --save-dev bun), or on the system (https://bun.sh)');
+}
+/**
+ * Write the icon of the application (ico) into the work directory
+ * @param work
+ * @param icon
+ */
+async function writeIcon(work, icon) {
+    const file = path.join(work, 'icon.ico');
+    await writeFile(file, icon);
+    return file;
+}
+/**
+ * Embed a bundle in the Node.js binary running the tool (the single executable applications of Node.js)
+ * @param bundle
+ * @param work
+ * @param file
+ * @param warnings
+ * @param icon the icon of the application (ico, Windows)
+ */
+async function embedInNode(bundle, work, file, warnings, icon) {
+    const assets = {};
+    if (bundle.native) {
+        //The files of the native packages, extracted by the executable at its first run
+        const { index, data } = packNativeTree(bundle.native);
+        assets[NATIVE_DATA_ASSET] = path.join(work, 'native.gz');
+        assets[NATIVE_INDEX_ASSET] = path.join(work, 'native.json');
+        await writeFile(assets[NATIVE_DATA_ASSET], data);
+        await writeFile(assets[NATIVE_INDEX_ASSET], JSON.stringify(index));
+    }
+    //The preparation blob of the bundle
+    const blob = path.join(work, 'main.blob');
+    await writeFile(path.join(work, 'sea-config.json'), JSON.stringify({
+        main: bundle.outfile,
+        output: blob,
+        disableExperimentalSEAWarning: true,
+        //The code cache of V8, made by the Node.js embedded: the bundle is not compiled again at start
+        useCodeCache: true,
+        useSnapshot: false,
+        assets: assets
+    }));
+    await must(process.execPath, ['--experimental-sea-config', 'sea-config.json'], work);
+    const { inject } = await import('postject');
+    await copyFile(process.execPath, file);
+    //The icon of the application replaces the one of node.exe, before the bundle is embedded
+    if (icon)
+        await setExecutableIcon(file, icon);
+    //The signature of macOS is removed before the binary is changed, then made again (ad hoc)
+    if (process.platform === 'darwin')
+        await must('codesign', ['--remove-signature', file], work);
+    await quiet(() => inject(file, 'NODE_SEA_BLOB', readFileSync(blob), { sentinelFuse: SEA_FUSE, machoSegmentName: process.platform === 'darwin' ? 'NODE_SEA' : undefined }));
+    if (process.platform === 'darwin')
+        await must('codesign', ['--sign', '-', '--force', file], work);
+    //The Authenticode signature of node.exe does not match the executable anymore
+    if (process.platform === 'win32')
+        warnings.push(`${path.basename(file)} has the signature of Node.js, which is not valid anymore: sign it (signtool) before distributing it`);
+    return `Node.js ${process.version}`;
+}
+/**
+ * Compile a bundle into an executable of Bun, in bytecode: the files of the native tree are embedded beside it, the
+ * executable gives their paths to the require of the bundle (BUN_NATIVE_ASSETS). As the executables of Node.js, it loads neither the .env
+ * nor the bunfig.toml of the directory it runs in; it reads the package.json of the packages it loads from the disk (the
+ * native tree, the modules left out)
+ * @param bun
+ * @param bundle
+ * @param work
+ * @param file
+ * @param icon the icon of the application (ico, Windows)
+ */
+async function compileWithBun(bun, bundle, work, file, icon) {
+    let entry = bundle.outfile;
+    if (bundle.native) {
+        const { index, data } = packNativeTree(bundle.native);
+        await writeFile(path.join(work, 'native.gz'), data);
+        await writeFile(path.join(work, 'native.json'), JSON.stringify(index));
+        entry = path.join(work, 'main.mjs');
+        await writeFile(entry, [
+            `import data from './native.gz' with {type: 'file'}`,
+            `import index from './native.json' with {type: 'file'}`,
+            `globalThis[Symbol.for(${JSON.stringify(BUN_NATIVE_ASSETS)})] = {data, index}`,
+            //Not imported: the bytecode of Bun is the one of CommonJS modules, without top-level await
+            `require(${JSON.stringify(`./${path.basename(bundle.outfile)}`)})`
+        ].join('\n'));
+    }
+    await must(bun.command, [
+        'build', entry, '--compile', '--outfile', file,
+        //Compiled to the bytecode of JavaScriptCore: it starts without parsing the bundle
+        '--bytecode',
+        '--compile-autoload-package-json', '--no-compile-autoload-dotenv', '--no-compile-autoload-bunfig',
+        //The icon of the application (Windows)
+        ...icon ? [`--windows-icon=${await writeIcon(work, icon)}`] : []
+    ], work);
+    return `Bun ${bun.version}`;
+}
+/**
+ * Make the executable of the application for the platform it is made on, which runs without Node.js installed: its bundle
+ * embedded in the Node.js binary running the tool (the single executable applications of Node.js), or compiled by Bun.
+ * The executables of the other platforms are made on them (their native addons are built there), as the applications are
+ * built there
+ * @param options
+ */
+export async function buildExecutable(options) {
+    const project = path.resolve(options.project);
+    const outdir = path.resolve(project, options.outdir ?? 'dist');
+    //Bun is found before the application is bundled
+    const bun = options.runtime === 'bun' ? await findBun(project) : undefined;
+    const work = await mkdtemp(path.join(os.tmpdir(), 'lakutata-executable-'));
+    try {
+        const directory = options.layout === 'directory';
+        //The directory of the executable: dist/<name>, its native tree and its resources written there (the links of
+        //the tree are the ones of their directory), the one of the executable alone otherwise
+        const manifest = JSON.parse(await readFile(path.join(project, 'package.json'), 'utf8'));
+        const target = directory ? path.join(outdir, programName(manifest)) : outdir;
+        if (directory)
+            await rm(target, { recursive: true, force: true });
+        await mkdir(target, { recursive: true });
+        const bundle = await bundleProject({
+            ...options,
+            outfile: path.join(work, 'main.cjs'),
+            native: directory ? 'directory' : 'embedded',
+            directories: { native: path.join(target, 'native'), resources: path.join(target, 'resources') }
+        });
+        //The executables of Node.js do not load the native addons on musl (postject rewrites their symbols): the ones of Bun do
+        if (!bun && detectLibc() === 'musl' && bundle.native?.files.some((file) => file.path.endsWith('.node')))
+            throw new Error('The executables of Node.js do not load native addons on musl (Alpine): make the executable with Bun (--runtime bun, or "runtime": "bun" of lakutata.installer)');
+        const file = path.join(target, process.platform === 'win32' ? `${bundle.name}.exe` : bundle.name);
+        await rm(file, { force: true });
+        //The native tree and the resources of the layout directory are not embedded
+        const embedded = directory ? { ...bundle, native: undefined } : bundle;
+        //Made and signed aside: codesign takes a directory holding "resources" for a bundle of macOS (its file system
+        //ignores the case), whose signature it writes beside the executable (_CodeSignature)
+        const made = path.join(work, 'executable', path.basename(file));
+        await mkdir(path.dirname(made), { recursive: true });
+        const warnings = [];
+        //The icon of the application: the one of the executables of Windows (the ones of macOS and Linux have none)
+        const application = applicationIcon(project, manifest);
+        const icon = application && process.platform === 'win32' ? icoFile(application.image) : undefined;
+        const runtime = bun ? await compileWithBun(bun, embedded, work, made, icon) : await embedInNode(embedded, work, made, warnings, icon);
+        await copyFile(made, file);
+        await chmod(file, 0o755);
+        return {
+            name: bundle.name,
+            file: file,
+            platform: `${process.platform === 'win32' ? 'win' : process.platform}-${process.arch}`,
+            size: (await stat(file)).size,
+            native: bundle.native?.modules ?? [],
+            resources: bundle.resources,
+            directory: directory ? target : undefined,
+            runtime: runtime,
+            warnings: warnings
+        };
+    }
+    finally {
+        await rm(work, { recursive: true, force: true });
+    }
+}