crossbind 2.0.0-beta.60 → 2.0.0-beta.63

package/src/actions/publishBinary.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import state from '../state/index.js';
+import replaceFile from '../utils/replaceFile.js';
+
+// paths.output defaults to paths.build, where a built command and its data already are; copying them onto
+// themselves would delete them first.
+const outputIsBuild = () => path.resolve(state.config.paths.output) === path.resolve(state.config.paths.build);
+
+export function publishNativeCommand(target) {
+    if (outputIsBuild()) return;
+    fs.mkdirSync(state.config.paths.output, { recursive: true });
+    replaceFile(`${state.config.paths.build}/${target.commandName}`, `${state.config.paths.output}/${target.commandName}`);
+}
+
+export function publishNodeAddon(target) {
+    if (outputIsBuild()) return;
+    fs.mkdirSync(state.config.paths.output, { recursive: true });
+    replaceFile(`${state.config.paths.build}/${target.addonName}`, `${state.config.paths.output}/${target.addonName}`);
+    fs.copyFileSync(`${state.config.paths.build}/${target.jsName}`, `${state.config.paths.output}/${target.jsName}`);
+}
+
+export function publishWasiCommand(target) {
+    if (outputIsBuild()) return;
+    fs.copyFileSync(`${state.config.paths.build}/${target.wasmName}`, `${state.config.paths.output}/${target.wasmName}`);
+    // Copied (not renamed) so a later fingerprint cache-hit still has the
+    // build-dir tree to serve from.
+    if (fs.existsSync(`${state.config.paths.build}/data`)) {
+        fs.rmSync(`${state.config.paths.output}/data`, { recursive: true, force: true });
+        fs.cpSync(`${state.config.paths.build}/data`, `${state.config.paths.output}/data`, { recursive: true });
+    }
+}

package/src/actions/run.js

@@ -1,24 +1,40 @@
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { withDirLockSync } from '../utils/dirLock.js';
 import pullDockerImage, { getDockerImage, getDockerContainerName, imageRoleFor } from '../utils/pullDockerImage.js';
 import getOsUserAndGroupId from '../utils/getOsUserAndGroupId.js';
 import replaceBasePathForDockerUtil from '../utils/replaceBasePathForDocker.js';
 import { DOCKER_RUN_SECURITY_ARGS } from '../utils/dockerSecurity.js';
+import { remoteExecParams, baseMount, referencedPaths } from '../utils/remoteRunner.js';
+import { runnerFor } from '../utils/selectRunner.js';
 import state from '../state/index.js';
 import { wasiCFlags, wasiCxxFlags, resolveWasiSdkPath, WASI_TARGET_TRIPLE } from '../utils/wasiToolchain.js';
+import { HOST_BUILT_PLATFORMS } from '../utils/targets.js';
+import { WASM_EXCEPTION_FLAGS } from '../utils/archiveFlags.js';
+import { ANDROID_NDK, ANDROID_API_LEVEL } from '../utils/androidToolchain.js';
+import { IOS_DEVELOPER_DIR, IOS_DEPLOYMENT_TARGET, XCODE_TOOLCHAIN_BIN } from '../utils/iosToolchain.js';
+import { LINUX_ARCHIVE_FLAGS, linuxBuildEnv, linuxToolchainFile } from '../utils/linuxToolchain.js';
+import { windowsBuildEnv, windowsToolchainFile } from '../utils/windowsToolchain.js';
+import {
+    DARWIN_CC, DARWIN_CXX, DARWIN_DEPLOYMENT_TARGET, DARWIN_HOST_PACKAGE_PREFIXES, DARWIN_TOOLS_BIN,
+} from '../utils/darwinToolchain.js';
 
 // Native builds can outrun Node's 1 MiB default pipe buffer; without a raised cap a successful build dies with ENOBUFS.
 const EXEC_MAX_BUFFER = 512 * 1024 * 1024;
-const CROSSCOMPILER_ARM64 = 'aarch64-linux-android33';
-const CROSSCOMPILER_x86_64 = 'x86_64-linux-android33';
-const ANDROID_NDK = '/opt/android-sdk/ndk/current';
+// Containers share the memory of Docker's VM, and one compile at full -j can take most of a desktop VM
+// (GDAL's bindings peak at 7.5 of 7.7 GiB), so the exclusive steps of parallel builds take turns.
+const DOCKER_COMPILE_LOCK = path.join(os.homedir(), '.crossbind', 'docker-compile.lock');
+const envParams = (env) => Object.entries(env).flatMap(([variable, value]) => ['-e', `${variable}=${value}`]);
+const CROSSCOMPILER_ARM64 = `aarch64-linux-android${ANDROID_API_LEVEL}`;
+const CROSSCOMPILER_x86_64 = `x86_64-linux-android${ANDROID_API_LEVEL}`;
 const t = `${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin`;
 const t2 = `${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64`;
 
-const iOSDevPath = '/Applications/Xcode.app/Contents/Developer';
-const iosBinPath = `${iOSDevPath}/Toolchains/XcodeDefault.xctoolchain/usr/bin`;
-const iosSdkPath = `${iOSDevPath}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk`;
-const iosSimSdkPath = `${iOSDevPath}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk`;
+const iosBinPath = XCODE_TOOLCHAIN_BIN;
+const iosSdkPath = `${IOS_DEVELOPER_DIR}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk`;
+const iosSimSdkPath = `${IOS_DEVELOPER_DIR}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk`;
 
 const androidParamsArm64 = [
     '-e',
@@ -62,8 +78,10 @@ const androidParamsX86_64 = [
     `CFLAGS=--sysroot=${t2}/sysroot`,
 ];
 
-// Xcode 27 refuses deployment targets below 15.0; React Native's own floor is 15.1.
-const IOS_DEPLOYMENT_TARGET = '15.1';
+const DARWIN_HOST_ARCH = process.arch === 'x64' ? 'x86_64' : 'arm64';
+// Apple's own tools only, which find the SDK through xcrun. A GNU ar earlier on the PATH (Homebrew's
+// binutils) writes archives that Apple's linker rejects.
+const DARWIN_BUILD_PATH = [DARWIN_TOOLS_BIN, '/bin', '/usr/sbin', '/sbin'].join(':');
 const IOS_HOST_FLAGS = `-arch arm64 -isysroot ${iosSdkPath} -fembed-bitcode`;
 const IOS_SIM_HOST_FLAGS = `-arch arm64 -isysroot ${iosSimSdkPath} -fembed-bitcode`;
 const IOS_IPHONE_PARAMS = ['-e', `CFLAGS="${IOS_HOST_FLAGS}"`, '-e', `CXXFLAGS="${IOS_HOST_FLAGS}"`, '-e', `LDFLAGS="${IOS_HOST_FLAGS}"`];
@@ -116,12 +134,16 @@ export default function run(program, params = [], platformPrefix = null, target
         fs.mkdirSync(buildPath, { recursive: true });
     }
 
-    // wasi is host-run only with a locally configured wasi-sdk; otherwise docker carries it.
+    const role = imageRoleFor(target);
+    const { runner, remote } = HOST_BUILT_PLATFORMS.includes(target?.platform) && program === null
+        ? { runner: 'LOCAL' }
+        : runnerFor(role, state.config.system);
+    const isDocker = runner === 'DOCKER_RUN' || runner === 'DOCKER_EXEC';
     const wasiHostSdk = target?.platform === 'wasi' ? resolveWasiSdkPath(state.config.system) : null;
-    if ((target?.platform !== 'ios' && !(target?.platform === 'wasi' && wasiHostSdk)) || program !== null) {
+    if (isDocker) {
         // Google ships the linux NDK for x86_64 only, so the android index carries no arm64 leaf;
         // pull its amd64 leaf by ref or an arm64 host finds no matching manifest.
-        pullDockerImage(imageRoleFor(target), target?.platform === 'android' ? 'linux/amd64' : undefined);
+        pullDockerImage(role, target?.platform === 'android' ? 'linux/amd64' : undefined);
     }
 
     let dProgram = program;
@@ -130,7 +152,7 @@ export default function run(program, params = [], platformPrefix = null, target
     if (program === null) {
         switch (target.platform) {
             case 'wasm':
-                platformParams = ['-e', 'CXXFLAGS=-fwasm-exceptions', '-e', 'CFLAGS=-fwasm-exceptions'];
+                platformParams = ['-e', `CXXFLAGS=${WASM_EXCEPTION_FLAGS.join(' ')}`, '-e', `CFLAGS=${WASM_EXCEPTION_FLAGS.join(' ')}`];
                 if (params[0].toLowerCase().includes('configure')) dProgram = 'emconfigure';
                 else if (params[0] === 'make') dProgram = 'emmake';
                 else if (params[0] === 'cmake') dProgram = 'emcmake';
@@ -179,7 +201,7 @@ export default function run(program, params = [], platformPrefix = null, target
                         ...dParams,
                         `-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake`,
                         `-DANDROID_ABI=${target.arch === 'x86_64' ? 'x86_64' : 'arm64-v8a'}`,
-                        '-DANDROID_PLATFORM=android-33',
+                        `-DANDROID_PLATFORM=android-${ANDROID_API_LEVEL}`,
                         `-DANDROID_NDK=${ANDROID_NDK}`,
                     ];
                 }
@@ -237,19 +259,61 @@ export default function run(program, params = [], platformPrefix = null, target
                     }
                 }
                 break;
+            case 'linux':
+            case 'linuxmusl': {
+                [dProgram, ...dParams] = params;
+                const archiveFlags = LINUX_ARCHIVE_FLAGS.join(' ');
+                platformParams = [
+                    ...envParams(linuxBuildEnv(target)),
+                    '-e', `CFLAGS=${archiveFlags}`, '-e', `CXXFLAGS=${archiveFlags}`,
+                ];
+                if (dProgram === 'cmake' && dParams[0] !== '--build' && dParams[0] !== '--install') {
+                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${linuxToolchainFile(target)}`];
+                }
+                break;
+            }
+            case 'win32': {
+                [dProgram, ...dParams] = params;
+                platformParams = envParams(windowsBuildEnv(target));
+                if (dProgram === 'cmake' && dParams[0] !== '--build' && dParams[0] !== '--install') {
+                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${windowsToolchainFile(target)}`];
+                }
+                break;
+            }
+            case 'darwin': {
+                [dProgram, ...dParams] = params;
+                const appleArch = target.arch === 'x64' ? 'x86_64' : 'arm64';
+                // None of the machine's own packages (DARWIN_HOST_PACKAGE_PREFIXES); the SDK's libraries stay visible.
+                platformParams = ['-e', 'PKG_CONFIG_LIBDIR='];
+                if (dProgram === 'cmake') {
+                    if (dParams[0] !== '--build' && dParams[0] !== '--install') {
+                        dParams = [
+                            ...dParams,
+                            `-DCMAKE_OSX_ARCHITECTURES=${appleArch}`,
+                            `-DCMAKE_OSX_DEPLOYMENT_TARGET=${DARWIN_DEPLOYMENT_TARGET}`,
+                            // Every archive ends up inside a loadable .node module.
+                            '-DCMAKE_POSITION_INDEPENDENT_CODE=ON',
+                            `-DCMAKE_IGNORE_PREFIX_PATH=${DARWIN_HOST_PACKAGE_PREFIXES.join(';')}`,
+                            // The other architecture's probes cannot run here, so CMake has to know it cross-compiles.
+                            ...(appleArch === DARWIN_HOST_ARCH ? [] : ['-DCMAKE_SYSTEM_NAME=Darwin', `-DCMAKE_SYSTEM_PROCESSOR=${appleArch}`]),
+                        ];
+                    }
+                } else {
+                    const flags = `-arch ${appleArch} -mmacosx-version-min=${DARWIN_DEPLOYMENT_TARGET}`;
+                    // A --host triple sends configure after prefixed compilers (aarch64-apple-darwin-cc) that
+                    // Xcode does not ship, and SQLite's autosetup has no fallback to the plain names.
+                    platformParams.push(
+                        '-e', `CC=${DARWIN_CC}`, '-e', `CXX=${DARWIN_CXX}`,
+                        '-e', `CFLAGS=${flags}`, '-e', `CXXFLAGS=${flags}`, '-e', `LDFLAGS=${flags}`,
+                    );
+                }
+                break;
+            }
             default:
         }
     }
 
     const env = {};
-    let runner = 'DOCKER';
-    if (
-        ((target?.platform === 'ios' || (target?.platform === 'wasi' && wasiHostSdk)) && program === null) ||
-        state.config.system.RUNNER === 'LOCAL'
-    ) {
-        runner = 'LOCAL';
-    }
-
     if (runner === 'LOCAL') {
         const allowedEnv = [
             '^PWD$',
@@ -288,9 +352,16 @@ export default function run(program, params = [], platformPrefix = null, target
         }
     }
 
+    const containerOptions = { cwd: buildPath, stdio: dockerOptions.console ? 'inherit' : ['ignore', 'pipe', 'pipe'], maxBuffer: EXEC_MAX_BUFFER };
     let fileExecParams;
     if (runner === 'LOCAL') {
         env.PATH = `/opt/homebrew/bin:${env.PATH}`;
+        // A package's *-config script on the PATH (nc-config, sfcgal-config, ...) would compile that
+        // package into the archives. cmake is the one host tool a darwin build takes from the PATH.
+        if (target?.platform === 'darwin') {
+            if (dProgram === 'cmake') dProgram = findExecutable('cmake', env.PATH) ?? dProgram;
+            env.PATH = DARWIN_BUILD_PATH;
+        }
 
         const options = {
             cwd: dockerOptions.workdir || buildPath,
@@ -299,17 +370,26 @@ export default function run(program, params = [], platformPrefix = null, target
             maxBuffer: EXEC_MAX_BUFFER,
         };
         fileExecParams = [dProgram, dParams, options];
-    } else if (runner === 'DOCKER') {
+    } else if (runner === 'REMOTE') {
+        fileExecParams = remoteExecParams({
+            remote,
+            mounts: [baseMount(state.config, { extraInputs: referencedPaths(state.config.paths.base, [dProgram, ...dParams, ...Object.values(env)]) })],
+            role,
+            image: getDockerImage(role, target?.platform === 'android' ? 'linux/amd64' : undefined),
+            cwd: replaceBasePathForDocker(dockerOptions.workdir || buildPath),
+            argv: [replaceBasePathForDocker(dProgram), ...replaceBasePathForDocker(dParams)],
+            env: replaceBasePathForDocker(env),
+        }, containerOptions);
+    } else {
         const dockerEnv = [];
         Object.entries(env).forEach(([key, value]) => {
             dockerEnv.push('-e', `${key}=${value}`);
         });
-        const options = { cwd: buildPath, stdio: dockerOptions.console ? 'inherit' : ['ignore', 'pipe', 'pipe'], maxBuffer: EXEC_MAX_BUFFER };
+        const options = containerOptions;
 
         let runnerParams;
         let imageOrContainer;
-        const role = imageRoleFor(target);
-        if (state.config.system.RUNNER === 'DOCKER_RUN') {
+        if (runner === 'DOCKER_RUN') {
             imageOrContainer = getDockerImage(role);
             runnerParams = ['run', '--rm', ...DOCKER_RUN_SECURITY_ARGS, '-v', `${state.config.paths.base}:/tmp/crossbind/live`];
             if (target?.platform === 'android') {
@@ -318,11 +398,9 @@ export default function run(program, params = [], platformPrefix = null, target
                 imageOrContainer = getDockerImage(role, 'linux/amd64');
                 runnerParams.push('--platform', 'linux/amd64');
             }
-        } else if (state.config.system.RUNNER === 'DOCKER_EXEC') {
+        } else {
             imageOrContainer = getDockerContainerName(state.config.paths.base, role);
             runnerParams = ['exec'];
-        } else {
-            throw new Error(`The runner ${state.config.system.RUNNER} is invalid.`);
         }
 
         const args = [
@@ -338,12 +416,15 @@ export default function run(program, params = [], platformPrefix = null, target
             ...replaceBasePathForDocker(dParams),
         ];
         fileExecParams = ['docker', args, options];
-    } else {
-        throw new Error(`The runner ${state.config.system.RUNNER} or command is invalid.`);
     }
 
     try {
-        execFileSync(...fileExecParams);
+        // A remote step takes no share of the local Docker VM, and its runner runs one step at a time.
+        if (isDocker && dockerOptions.exclusive) {
+            withDirLockSync(DOCKER_COMPILE_LOCK, () => execFileSync(...fileExecParams), { heldBy: 'another build is compiling in Docker' });
+        } else {
+            execFileSync(...fileExecParams);
+        }
     } catch (e) {
         if (e?.stdout?.length) console.log(e.stdout.toString());
         if (e?.stderr?.length) console.error(e.stderr.toString());
@@ -351,6 +432,17 @@ export default function run(program, params = [], platformPrefix = null, target
     }
 }
 
+function findExecutable(name, searchPath) {
+    return searchPath.split(':').map((dir) => `${dir}/${name}`).find((file) => {
+        try {
+            fs.accessSync(file, fs.constants.X_OK);
+            return true;
+        } catch {
+            return false;
+        }
+    });
+}
+
 function replaceBasePathForDocker(data) {
     return replaceBasePathForDockerUtil(data, state.config.paths.base);
 }