@gjsify/cli 0.64.8 → 0.64.10

package/lib/commands/ship.js

@@ -35,6 +35,7 @@ import { describeExit, spawnToCompletion } from '../utils/spawn.js';
 import { buildDeb } from '../utils/ship/deb.js';
 import { deriveDepends, warnAboutGjsFloor, warnAboutNodeFloor } from '../utils/ship/depends.js';
 import { isGtkRuntimeTarget, stageAppRuntime } from '../utils/ship/app-runtime.js';
+import { resolveNodeModulesClosure, stageNodeModules } from '../utils/ship/node-modules.js';
 import { acquireGtkRuntime } from '../utils/ship/gtk-runtime-acquire.js';
 import { discoverPayload } from '../utils/ship/discover.js';
 import { appDirFor, appDirPayload, appImageHostRequirements, appImageRuntimeNotice, buildAppImage, findPinnedRuntime, stampAppDirTimes, } from '../utils/ship/appimage.js';
@@ -44,7 +45,7 @@ import { buildFlatpakBundle } from '../utils/ship/flatpak.js';
 import { iconSizes, resolveAppIcon } from '../utils/ship/icons.js';
 import { localizeMetadata } from '../utils/ship/localize-metadata.js';
 import { assertHostCanFinish, assertKindCanPack, assertToolsInstalled, canCarryTicket, configuredFormats, defaultFormatIds, formatIdsFor, resolveFormats, windowsProgramDirName, FORMAT_IDS, FORMATS, } from '../utils/ship/formats.js';
-import { assertLayoutSupportsArch, hostLayout, layoutForOs, place, placeStage, resolveLayout, windowsGuiLauncherPath, LAYOUT_NAMES, LAYOUTS, } from '../utils/ship/layout.js';
+import { assertLayoutSupportsArch, hostLayout, layoutForOs, place, placeStage, resolveLayout, payloadArchExemptions, LAYOUT_NAMES, } from '../utils/ship/layout.js';
 import { isNodeRuntimeTarget, resolveNodeRuntime } from '../utils/ship/node-runtime.js';
 import { isUnder, SHARE } from '../utils/ship/share-dirs.js';
 import { compileSchemasForPayload, compileSchemasForStage } from '../utils/ship/schemas.js';
@@ -416,6 +417,7 @@ async function assemble(args) {
     // carrying a second copy inside the payload would be a private interpreter no
     // distribution update ever patches.
     const runtime = await resolveCarriedRuntime(layout, settings, projectDir);
+    carryNodeModules(runtime, layout, settings, projectDir, ship.nodeModules, ship.nodeModulesExclude);
     const stageInputs = {
         bundleFiles: discovered.bundleFiles,
         launcher: renderLauncher(settings, basename(settings.bundlePath), layout, runtime.launcher),
@@ -510,7 +512,7 @@ async function assemble(args) {
     // stub unconditionally (it used to skip arm64 entirely), so an arm64 stage
     // legitimately carries one x64 PE beside its arm64 payload.
     assertPayloadMatchesArch(readStage([stageDir], staged), settings.arch, {
-        exempt: layout === LAYOUTS.windows ? new Set([windowsGuiLauncherPath(settings)]) : undefined,
+        exempt: payloadArchExemptions(layout.os, settings),
     });
     // What the map carried into a layout that has no install step for it.
     //
@@ -802,6 +804,35 @@ async function resolveCarriedRuntime(layout, settings, projectDir) {
         ...(acquired.kind === 'failed' ? { gtkFetchFailure: acquired.reason } : {}),
     });
 }
+/**
+ * `gjsify.ship.nodeModules`: the npm packages an `--app node` bundle imports and cannot contain,
+ * staged beside the bundle with what they need at runtime (`utils/ship/node-modules.ts`).
+ *
+ * Only for a node payload: a GJS bundle inlines or rewrites them, so one declaration can serve a
+ * project that ships GJS on Linux and Node on macOS and Windows. Mutates `runtime` because the
+ * files take the same route as the carried interpreter — already stage-relative, past the planner.
+ */
+function carryNodeModules(runtime, layout, settings, projectDir, declared, exclude) {
+    if (declared === undefined || declared.length === 0 || settings.app !== 'node')
+        return;
+    const closure = resolveNodeModulesClosure(declared, projectDir, {
+        target: { os: layout.os, cpu: settings.arch },
+        exclude,
+    });
+    runtime.files.push(...stageNodeModules(closure, layout.dirs(settings).bundle));
+    runtime.found.push(`node_modules: ${closure.packages.length} package(s) for ${declared.join(', ')}`);
+    if (closure.excluded.length > 0) {
+        console.log(`${LOG} node_modules: left out by gjsify.ship.nodeModulesExclude: ${closure.excluded.join(', ')}`);
+    }
+    if (closure.skippedForTarget.length > 0) {
+        console.log(`${LOG} node_modules: left out, installed but built for another platform than ${layout.os}-${settings.arch}: ` +
+            closure.skippedForTarget.join(', '));
+    }
+    if (closure.skippedOptional.length > 0) {
+        console.log(`${LOG} node_modules: ${closure.skippedOptional.length} optional dependenc(ies) not installed, ` +
+            `so not carried (other platforms' builds): ${closure.skippedOptional.join(', ')}`);
+    }
+}
 /** How long a fetched runtime stays in the dlx cache — the same week `gjsify showcase` uses. */
 const DLX_CACHE_MAX_AGE_MIN = 60 * 24 * 7;
 /**
@@ -979,7 +1010,9 @@ async function packOne(input) {
     // asserting afterwards would compare against a label that no longer names an
     // architecture, and the mismatch that matters is exactly the one on a payload
     // that IS architecture-specific.
-    assertPayloadMatchesArch(assembled, input.arch);
+    assertPayloadMatchesArch(assembled, input.arch, {
+        exempt: payloadArchExemptions(format.layoutOs, settings),
+    });
     // The second half of the same idea, on the other label this artifact carries:
     // something declares which interpreter runs the app, and until now nothing
     // compared it to the launcher that will. They CAN disagree — a stage assembled

package/lib/types/config-data.d.ts

@@ -606,6 +606,31 @@ export interface ConfigDataShip extends AppMetadata {
      * bundle seal and `dlopen` both refuse. A target with no key stages nothing.
      */
     bundledTypelibs?: string[] | ShipBundledTypelibsOptions;
+    /**
+     * npm packages the `--app node` bundle imports but cannot contain, by NAME — e.g.
+     * `["node-llama-cpp"]`.
+     *
+     * An `--app node` build keeps every Node-API addon package external (an addon finds its
+     * `.node` relative to its own files), so the bundle `import`s it by name at runtime. A `.app`
+     * or program directory has no `npm install`; without this entry the artifact fails with
+     * `ERR_MODULE_NOT_FOUND` before the program starts. Each name is staged into the bundle's
+     * `node_modules/` with its `dependencies` and `optionalDependencies`, resolved from the
+     * project the way Node would, at the paths they are installed at (two versions of one package
+     * stay nested). An optional dependency that is not installed is skipped, which is how a
+     * per-platform addon package is chosen: `gjsify install --os <os> --cpu <cpu>` before `ship`.
+     * A required one that is missing is an error.
+     *
+     * Ignored for a GJS payload, so one declaration serves a project that ships GJS on Linux and
+     * Node on macOS and Windows. `@gjsify/node-gi` is not listed here; `ship` carries it itself.
+     */
+    nodeModules?: string[];
+    /**
+     * Packages {@link ConfigDataShip.nodeModules} leaves out wherever they turn up in the closure —
+     * for an optional dependency the project does not want to ship, e.g. the CUDA and Vulkan
+     * builds of an inference addon (`@node-llama-cpp/win-x64-cuda`, hundreds of MB) when the
+     * program only runs on the CPU. Names a declared package itself are not affected.
+     */
+    nodeModulesExclude?: string[];
     /**
      * Additions for ONE package format — `deb` or `rpm` — on top of everything above.
      *

package/lib/utils/ship/layout.d.ts

@@ -330,6 +330,16 @@ export declare function launcherPath(layout: Layout, identity: LayoutIdentity):
  * is not there, at exit 0, with a window that never appears.
  */
 export declare function windowsGuiLauncherPath(identity: LayoutIdentity): string;
+/**
+ * The payload paths exempt from the architecture check for one OS (`assertPayloadMatchesArch`).
+ *
+ * On windows that is the GUI-launcher stub, x86-64 machine code on every windows arch by design.
+ * ONE function for the two places that read the stage back — the assemble phase and the pack
+ * phase — because the pack phase once had its own call without the exemption: an arm64 stage
+ * assembled fine and every package refused it with "stamped.exe is built for x64". `undefined`
+ * for an OS with no such file.
+ */
+export declare function payloadArchExemptions(os: HostOs, identity: LayoutIdentity): ReadonlySet<string> | undefined;
 /**
  * What the windows GUI launcher calls the file it writes under `%TEMP%`.
  *

package/lib/utils/ship/layout.js

@@ -552,6 +552,18 @@ export function windowsGuiLauncherPath(identity) {
     }
     return posix.join(layout.dirs(identity).launcher, `${identity.binaryName}.exe`);
 }
+/**
+ * The payload paths exempt from the architecture check for one OS (`assertPayloadMatchesArch`).
+ *
+ * On windows that is the GUI-launcher stub, x86-64 machine code on every windows arch by design.
+ * ONE function for the two places that read the stage back — the assemble phase and the pack
+ * phase — because the pack phase once had its own call without the exemption: an arm64 stage
+ * assembled fine and every package refused it with "stamped.exe is built for x64". `undefined`
+ * for an OS with no such file.
+ */
+export function payloadArchExemptions(os, identity) {
+    return os === 'win32' ? new Set([windowsGuiLauncherPath(identity)]) : undefined;
+}
 /**
  * What the windows GUI launcher calls the file it writes under `%TEMP%`.
  *

package/lib/utils/ship/node-modules.d.ts

@@ -0,0 +1,39 @@
+import type { StagedFile } from './types.js';
+/** One package of the closure, ready to stage. */
+export interface NodeModulePackage {
+    name: string;
+    version: string;
+    /** Where the files are read from — the installed directory. */
+    dir: string;
+    /** POSIX path below the staged `node_modules/`: `yargs`, `a/node_modules/b`, `@scope/pkg`. */
+    dest: string;
+}
+/** The platform the artifact is for, spelled like npm's `os` / `cpu` fields (`darwin`, `win32`; `arm64`, `x64`). */
+export interface NodeModulesTarget {
+    os: string;
+    cpu: string;
+}
+export interface NodeModulesOptions {
+    /** Platform filter, see {@link NodeModulesTarget}. Without one nothing is filtered by `os` / `cpu`. */
+    target?: NodeModulesTarget;
+    /** Package names left out wherever they appear below a declared one — `gjsify.ship.nodeModulesExclude`. */
+    exclude?: readonly string[];
+}
+export interface NodeModulesClosure {
+    packages: NodeModulePackage[];
+    /** Optional dependencies that are not installed — expected for other platforms, listed for the log. */
+    skippedOptional: string[];
+    /** Dependencies installed here but built for another platform, named by their `os` / `cpu` fields. */
+    skippedForTarget: string[];
+    /** Dependencies the project asked to leave out, and that were reached. */
+    excluded: string[];
+}
+/**
+ * Resolve `names` and everything they need at runtime, from `cwd`.
+ *
+ * Throws on a name that is not installed, a required dependency that is not, and two different
+ * installed copies that would land on one staged path.
+ */
+export declare function resolveNodeModulesClosure(names: readonly string[], cwd: string, options?: NodeModulesOptions): NodeModulesClosure;
+/** The closure as stage-relative files, under `<bundleDir>/node_modules/`. */
+export declare function stageNodeModules(closure: NodeModulesClosure, bundleDir: string): StagedFile[];

package/lib/utils/ship/node-modules.js

@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: MIT
+// npm packages an `--app node` bundle imports but cannot contain: found, then staged as a TREE.
+//
+// WHY THIS EXISTS. `rolldown-plugin-gjsify`'s `nodeNativeExternalPlugin` keeps every Node-API
+// addon package EXTERNAL in an `--app node` bundle, because an addon finds its `.node` relative
+// to its own files and a bundle moves them ("an `--app node` bundle of a native dependency needs
+// `node_modules` at runtime, like any `npm install`-ed program"). A shipped `.app` or program
+// directory has no `npm install`. Until this module the only package `ship` carried was
+// `@gjsify/node-gi`, by name — so a project with one more native dependency produced an artifact
+// that fails at its first `import` on every machine but the author's. Measured on a real consumer
+// (Stamped, `node-llama-cpp`): the released macOS `.app` exited with
+// `ERR_MODULE_NOT_FOUND: Cannot find package 'node-llama-cpp'` before opening a window, and no
+// CI step noticed because nothing ever started the artifact.
+//
+// THE DECLARATION. `gjsify.ship.nodeModules` lists package NAMES, nothing else. The names are the
+// entry points the bundle imports; this module adds everything they need at runtime.
+//
+// THE CLOSURE is node's own resolution rule applied to the installed tree: for each package,
+// `dependencies` and `optionalDependencies`, each found by walking `node_modules` upward from the
+// package that asks for it. A missing optional dependency is skipped silently — that is the
+// per-platform filter (`@node-llama-cpp/mac-arm64-metal` is optional; `gjsify install --os
+// darwin --cpu arm64` is what puts the right one on disk; one that IS on disk but whose `os` /
+// `cpu` fields exclude the artifact's target is left out too, since installs accumulate) — and a missing required one is an
+// error naming the chain, because a package without its dependency is the very artifact this
+// exists to prevent.
+//
+// THE LAYOUT IS THE INSTALLED ONE. Each package lands at the path it was FOUND at, relative to
+// the first `node_modules` in that path: `node_modules/a/node_modules/b` stays nested, which is
+// how two versions of `yargs` coexist. Flattening would be a guess about which one wins.
+//
+// NOT INCLUDED: `@gjsify/node-gi`, which `app-runtime.ts` stages as a trimmed tree of its own, and
+// any nested `node_modules` that is not reached through a declared dependency (an npm install
+// leaves dev-only trees there).
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { dirname, join, posix, relative, sep } from 'node:path';
+import { isExecutableAsset } from './plan.js';
+/** The package this module leaves to `app-runtime.ts`. */
+const LEFT_TO_APP_RUNTIME = '@gjsify/node-gi';
+function readManifest(dir) {
+    try {
+        return JSON.parse(readFileSync(join(dir, 'package.json'), 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+/** `<from>/node_modules/<name>`, then each ancestor's — node's lookup, on the file system alone. */
+function findInstalled(name, from) {
+    let dir = from;
+    for (;;) {
+        const candidate = join(dir, 'node_modules', ...name.split('/'));
+        if (existsSync(join(candidate, 'package.json')))
+            return candidate;
+        const parent = dirname(dir);
+        if (parent === dir)
+            return null;
+        dir = parent;
+    }
+}
+/** npm's rule for `os` / `cpu`: a `!x` entry blocks `x`; otherwise, if any plain entry exists, `value` must be one. */
+function allows(field, value) {
+    const list = typeof field === 'string' ? [field] : (field ?? []);
+    if (list.includes(`!${value}`))
+        return false;
+    const positive = list.filter((entry) => !entry.startsWith('!'));
+    return positive.length === 0 || positive.includes(value);
+}
+/** The path below the first `node_modules` segment, POSIX-separated, or `null` when there is none. */
+function destinationOf(found) {
+    const parts = found.split(sep);
+    const at = parts.indexOf('node_modules');
+    return at === -1 || at === parts.length - 1 ? null : parts.slice(at + 1).join('/');
+}
+/**
+ * Resolve `names` and everything they need at runtime, from `cwd`.
+ *
+ * Throws on a name that is not installed, a required dependency that is not, and two different
+ * installed copies that would land on one staged path.
+ */
+export function resolveNodeModulesClosure(names, cwd, options = {}) {
+    const { target, exclude = [] } = options;
+    const packages = new Map();
+    const skippedOptional = [];
+    const skippedForTarget = [];
+    const excluded = [];
+    const visit = (name, from, via, optional) => {
+        if (name === LEFT_TO_APP_RUNTIME)
+            return;
+        if (via.length > 0 && exclude.includes(name)) {
+            if (!excluded.includes(name))
+                excluded.push(name);
+            return;
+        }
+        const found = findInstalled(name, from);
+        if (found === null) {
+            if (optional) {
+                if (!skippedOptional.includes(name))
+                    skippedOptional.push(name);
+                return;
+            }
+            throw new Error(`gjsify ship: \`${name}\` is not installed${via.length > 0 ? ` (needed by ${via.join(' > ')})` : ''}. ` +
+                (via.length === 0
+                    ? '`gjsify.ship.nodeModules` names it, so the artifact must carry it — install it, or remove the entry.'
+                    : 'A package without its required dependency is an artifact that fails at its first import.'));
+        }
+        const dest = destinationOf(found);
+        if (dest === null) {
+            throw new Error(`gjsify ship: ${found} is not inside a node_modules directory, so it has no place to be staged.`);
+        }
+        const dir = realpathSync(found);
+        const manifest = readManifest(dir);
+        if (manifest === null)
+            throw new Error(`gjsify ship: ${join(dir, 'package.json')} is unreadable.`);
+        // A dependency built for another platform may well be on disk: `gjsify install --os` adds to a
+        // tree, it does not clear it. Carrying it would put a foreign binary in the artifact.
+        if (via.length > 0 &&
+            target !== undefined &&
+            !(allows(manifest.os, target.os) && allows(manifest.cpu, target.cpu))) {
+            if (!skippedForTarget.includes(name))
+                skippedForTarget.push(name);
+            return;
+        }
+        const existing = packages.get(dest);
+        if (existing !== undefined) {
+            if (existing.dir !== dir) {
+                throw new Error(`gjsify ship: two installed copies of \`${name}\` would both be staged at node_modules/${dest} ` +
+                    `(${existing.dir} and ${dir}).`);
+            }
+            return;
+        }
+        packages.set(dest, { name, version: manifest.version ?? '0.0.0', dir, dest });
+        const chain = [...via, name];
+        for (const dep of Object.keys(manifest.dependencies ?? {}))
+            visit(dep, found, chain, false);
+        for (const dep of Object.keys(manifest.optionalDependencies ?? {}))
+            visit(dep, found, chain, true);
+    };
+    for (const name of names)
+        visit(name, cwd, [], false);
+    return {
+        packages: [...packages.values()].sort((a, b) => (a.dest < b.dest ? -1 : 1)),
+        skippedOptional: skippedOptional.sort(),
+        skippedForTarget: skippedForTarget.sort(),
+        excluded: excluded.sort(),
+    };
+}
+/**
+ * Every file of one package, package-relative, without the `node_modules` directories inside it —
+ * those are separate packages of the closure, or not wanted at all.
+ */
+function listPackageFiles(root) {
+    const out = [];
+    const walk = (current) => {
+        for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) {
+            const full = join(current, entry.name);
+            if (entry.isDirectory()) {
+                if (entry.name !== 'node_modules')
+                    walk(full);
+            }
+            else if (entry.isSymbolicLink()) {
+                throw new Error(`gjsify ship: ${full} is a symlink. The payload has to be self-contained — a package that ` +
+                    'links inside itself needs the link replaced by the file.');
+            }
+            else if (entry.isFile()) {
+                out.push(posix.join(...relative(root, full).split(sep)));
+            }
+        }
+    };
+    walk(root);
+    return out;
+}
+/** 0755 for what the source marks executable or is a shared library, 0644 for the rest. */
+function modeOf(rel, abs) {
+    if (isExecutableAsset(rel))
+        return 0o755;
+    return (statSync(abs).mode & 0o111) !== 0 ? 0o755 : 0o644;
+}
+/** The closure as stage-relative files, under `<bundleDir>/node_modules/`. */
+export function stageNodeModules(closure, bundleDir) {
+    const files = [];
+    for (const pkg of closure.packages) {
+        for (const rel of listPackageFiles(pkg.dir)) {
+            const abs = join(pkg.dir, ...rel.split('/'));
+            files.push({
+                path: posix.join(bundleDir, 'node_modules', pkg.dest, rel),
+                mode: modeOf(rel, abs),
+                source: { kind: 'file', path: abs },
+            });
+        }
+    }
+    return files;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gjsify/cli",
-    "version": "0.64.8",
+    "version": "0.64.10",
     "description": "CLI for Gjsify",
     "type": "module",
     "main": "lib/index.js",
@@ -142,22 +142,22 @@
         "cli"
     ],
     "dependencies": {
-        "@gjsify/blueprint": "^0.64.8",
-        "@gjsify/buffer": "^0.64.8",
-        "@gjsify/create-app": "^0.64.8",
-        "@gjsify/node-globals": "^0.64.8",
-        "@gjsify/node-polyfills": "^0.64.8",
-        "@gjsify/npm-registry": "^0.64.8",
-        "@gjsify/resolve-npm": "^0.64.8",
-        "@gjsify/rolldown-plugin-gjsify": "^0.64.8",
-        "@gjsify/vite-plugin-blueprint": "^0.64.8",
-        "@gjsify/rolldown-plugin-pnp": "^0.64.8",
-        "@gjsify/semver": "^0.64.8",
-        "@gjsify/tar": "^0.64.8",
-        "@gjsify/tsc": "^0.64.8",
-        "@gjsify/utils": "^0.64.8",
-        "@gjsify/web-polyfills": "^0.64.8",
-        "@gjsify/workspace": "^0.64.8",
+        "@gjsify/blueprint": "^0.64.10",
+        "@gjsify/buffer": "^0.64.10",
+        "@gjsify/create-app": "^0.64.10",
+        "@gjsify/node-globals": "^0.64.10",
+        "@gjsify/node-polyfills": "^0.64.10",
+        "@gjsify/npm-registry": "^0.64.10",
+        "@gjsify/resolve-npm": "^0.64.10",
+        "@gjsify/rolldown-plugin-gjsify": "^0.64.10",
+        "@gjsify/vite-plugin-blueprint": "^0.64.10",
+        "@gjsify/rolldown-plugin-pnp": "^0.64.10",
+        "@gjsify/semver": "^0.64.10",
+        "@gjsify/tar": "^0.64.10",
+        "@gjsify/tsc": "^0.64.10",
+        "@gjsify/utils": "^0.64.10",
+        "@gjsify/web-polyfills": "^0.64.10",
+        "@gjsify/workspace": "^0.64.10",
         "acorn": "^8.17.0",
         "cosmiconfig": "^9.0.2",
         "get-tsconfig": "^4.14.0",
@@ -166,13 +166,13 @@
         "yargs": "^18.0.0"
     },
     "devDependencies": {
-        "@gjsify/unit": "^0.64.8",
+        "@gjsify/unit": "^0.64.10",
         "@types/yargs": "^17.0.35",
         "typescript": "^6.0.3"
     },
     "peerDependencies": {
-        "@gjsify/oxfmt-native": "^0.64.8",
-        "@gjsify/rolldown-native": "^0.64.8"
+        "@gjsify/oxfmt-native": "^0.64.10",
+        "@gjsify/rolldown-native": "^0.64.10"
     },
     "peerDependenciesMeta": {
         "@gjsify/oxfmt-native": {