crossbind 2.0.0-beta.62 → 2.0.0-beta.64

package/src/actions/run.js

@@ -1,25 +1,40 @@
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { withDirLockSync } from '../utils/dirLock.js';
 import pullDockerImage, { getDockerImage, getDockerContainerName, imageRoleFor } from '../utils/pullDockerImage.js';
 import getOsUserAndGroupId from '../utils/getOsUserAndGroupId.js';
 import replaceBasePathForDockerUtil from '../utils/replaceBasePathForDocker.js';
 import { DOCKER_RUN_SECURITY_ARGS } from '../utils/dockerSecurity.js';
+import { remoteExecParams, baseMount, referencedPaths } from '../utils/remoteRunner.js';
+import { runnerFor } from '../utils/selectRunner.js';
 import state from '../state/index.js';
 import { wasiCFlags, wasiCxxFlags, resolveWasiSdkPath, WASI_TARGET_TRIPLE } from '../utils/wasiToolchain.js';
 import { HOST_BUILT_PLATFORMS } from '../utils/targets.js';
+import { WASM_EXCEPTION_FLAGS } from '../utils/archiveFlags.js';
+import { ANDROID_NDK, ANDROID_API_LEVEL } from '../utils/androidToolchain.js';
+import { IOS_DEVELOPER_DIR, IOS_DEPLOYMENT_TARGET, XCODE_TOOLCHAIN_BIN } from '../utils/iosToolchain.js';
+import { LINUX_ARCHIVE_FLAGS, linuxBuildEnv, linuxToolchainFile } from '../utils/linuxToolchain.js';
+import { windowsBuildEnv, windowsToolchainFile } from '../utils/windowsToolchain.js';
+import {
+    DARWIN_CC, DARWIN_CXX, DARWIN_DEPLOYMENT_TARGET, DARWIN_HOST_PACKAGE_PREFIXES, DARWIN_TOOLS_BIN,
+} from '../utils/darwinToolchain.js';
 
 // Native builds can outrun Node's 1 MiB default pipe buffer; without a raised cap a successful build dies with ENOBUFS.
 const EXEC_MAX_BUFFER = 512 * 1024 * 1024;
-const CROSSCOMPILER_ARM64 = 'aarch64-linux-android33';
-const CROSSCOMPILER_x86_64 = 'x86_64-linux-android33';
-const ANDROID_NDK = '/opt/android-sdk/ndk/current';
+// Containers share the memory of Docker's VM, and one compile at full -j can take most of a desktop VM
+// (GDAL's bindings peak at 7.5 of 7.7 GiB), so the exclusive steps of parallel builds take turns.
+const DOCKER_COMPILE_LOCK = path.join(os.homedir(), '.crossbind', 'docker-compile.lock');
+const envParams = (env) => Object.entries(env).flatMap(([variable, value]) => ['-e', `${variable}=${value}`]);
+const CROSSCOMPILER_ARM64 = `aarch64-linux-android${ANDROID_API_LEVEL}`;
+const CROSSCOMPILER_x86_64 = `x86_64-linux-android${ANDROID_API_LEVEL}`;
 const t = `${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin`;
 const t2 = `${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64`;
 
-const iOSDevPath = '/Applications/Xcode.app/Contents/Developer';
-const iosBinPath = `${iOSDevPath}/Toolchains/XcodeDefault.xctoolchain/usr/bin`;
-const iosSdkPath = `${iOSDevPath}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk`;
-const iosSimSdkPath = `${iOSDevPath}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk`;
+const iosBinPath = XCODE_TOOLCHAIN_BIN;
+const iosSdkPath = `${IOS_DEVELOPER_DIR}/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk`;
+const iosSimSdkPath = `${IOS_DEVELOPER_DIR}/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk`;
 
 const androidParamsArm64 = [
     '-e',
@@ -63,20 +78,10 @@ const androidParamsX86_64 = [
     `CFLAGS=--sysroot=${t2}/sysroot`,
 ];
 
-// Xcode 27 refuses deployment targets below 15.0; React Native's own floor is 15.1.
-const IOS_DEPLOYMENT_TARGET = '15.1';
-// Node 22, the oldest supported line, needs macOS 11.
-const DARWIN_DEPLOYMENT_TARGET = '11.0';
 const DARWIN_HOST_ARCH = process.arch === 'x64' ? 'x86_64' : 'arm64';
-const DARWIN_HOST_PACKAGE_PREFIXES = ['/opt/homebrew', '/usr/local', '/opt/local'];
 // Apple's own tools only, which find the SDK through xcrun. A GNU ar earlier on the PATH (Homebrew's
 // binutils) writes archives that Apple's linker rejects.
-const DARWIN_BUILD_PATH = '/usr/bin:/bin:/usr/sbin:/sbin';
-// The linux image's clang wrappers and CMake toolchain files, one per target triple.
-const LINUX_TOOLCHAIN = '/opt/crossbind/linux';
-// The windows image's llvm-mingw and its CMake toolchain files, one per target triple.
-const LLVM_MINGW = '/opt/llvm-mingw';
-const WINDOWS_TOOLCHAIN = '/opt/crossbind/windows';
+const DARWIN_BUILD_PATH = [DARWIN_TOOLS_BIN, '/bin', '/usr/sbin', '/sbin'].join(':');
 const IOS_HOST_FLAGS = `-arch arm64 -isysroot ${iosSdkPath} -fembed-bitcode`;
 const IOS_SIM_HOST_FLAGS = `-arch arm64 -isysroot ${iosSimSdkPath} -fembed-bitcode`;
 const IOS_IPHONE_PARAMS = ['-e', `CFLAGS="${IOS_HOST_FLAGS}"`, '-e', `CXXFLAGS="${IOS_HOST_FLAGS}"`, '-e', `LDFLAGS="${IOS_HOST_FLAGS}"`];
@@ -129,12 +134,16 @@ export default function run(program, params = [], platformPrefix = null, target
         fs.mkdirSync(buildPath, { recursive: true });
     }
 
-    // wasi is host-run only with a locally configured wasi-sdk; otherwise docker carries it.
+    const role = imageRoleFor(target);
+    const { runner, remote } = HOST_BUILT_PLATFORMS.includes(target?.platform) && program === null
+        ? { runner: 'LOCAL' }
+        : runnerFor(role, state.config.system);
+    const isDocker = runner === 'DOCKER_RUN' || runner === 'DOCKER_EXEC';
     const wasiHostSdk = target?.platform === 'wasi' ? resolveWasiSdkPath(state.config.system) : null;
-    if ((!HOST_BUILT_PLATFORMS.includes(target?.platform) && !(target?.platform === 'wasi' && wasiHostSdk)) || program !== null) {
+    if (isDocker) {
         // Google ships the linux NDK for x86_64 only, so the android index carries no arm64 leaf;
         // pull its amd64 leaf by ref or an arm64 host finds no matching manifest.
-        pullDockerImage(imageRoleFor(target), target?.platform === 'android' ? 'linux/amd64' : undefined);
+        pullDockerImage(role, target?.platform === 'android' ? 'linux/amd64' : undefined);
     }
 
     let dProgram = program;
@@ -143,7 +152,7 @@ export default function run(program, params = [], platformPrefix = null, target
     if (program === null) {
         switch (target.platform) {
             case 'wasm':
-                platformParams = ['-e', 'CXXFLAGS=-fwasm-exceptions', '-e', 'CFLAGS=-fwasm-exceptions'];
+                platformParams = ['-e', `CXXFLAGS=${WASM_EXCEPTION_FLAGS.join(' ')}`, '-e', `CFLAGS=${WASM_EXCEPTION_FLAGS.join(' ')}`];
                 if (params[0].toLowerCase().includes('configure')) dProgram = 'emconfigure';
                 else if (params[0] === 'make') dProgram = 'emmake';
                 else if (params[0] === 'cmake') dProgram = 'emcmake';
@@ -192,7 +201,7 @@ export default function run(program, params = [], platformPrefix = null, target
                         ...dParams,
                         `-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake`,
                         `-DANDROID_ABI=${target.arch === 'x86_64' ? 'x86_64' : 'arm64-v8a'}`,
-                        '-DANDROID_PLATFORM=android-33',
+                        `-DANDROID_PLATFORM=android-${ANDROID_API_LEVEL}`,
                         `-DANDROID_NDK=${ANDROID_NDK}`,
                     ];
                 }
@@ -250,46 +259,31 @@ export default function run(program, params = [], platformPrefix = null, target
                     }
                 }
                 break;
-            case 'linux': {
+            case 'linux':
+            case 'linuxmusl': {
                 [dProgram, ...dParams] = params;
-                const triple = target.arch === 'x64' ? 'x86_64-linux-gnu' : 'aarch64-linux-gnu';
-                const tool = (name) => `${LINUX_TOOLCHAIN}/bin/${triple}-${name}`;
-                // The image's own .pc files describe its libraries, not the sysroot's. Every archive
-                // ends up inside a loadable .node module, so -fPIC holds even where a project turns
-                // position-independent code off for static builds, as GDAL does.
+                const archiveFlags = LINUX_ARCHIVE_FLAGS.join(' ');
                 platformParams = [
-                    '-e', `CC=${tool('clang')}`, '-e', `CXX=${tool('clang++')}`,
-                    '-e', `AR=${tool('ar')}`, '-e', `RANLIB=${tool('ranlib')}`,
-                    '-e', `NM=${tool('nm')}`, '-e', `STRIP=${tool('strip')}`,
-                    '-e', 'PKG_CONFIG_LIBDIR=',
-                    '-e', 'CFLAGS=-fPIC', '-e', 'CXXFLAGS=-fPIC',
+                    ...envParams(linuxBuildEnv(target)),
+                    '-e', `CFLAGS=${archiveFlags}`, '-e', `CXXFLAGS=${archiveFlags}`,
                 ];
                 if (dProgram === 'cmake' && dParams[0] !== '--build' && dParams[0] !== '--install') {
-                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${LINUX_TOOLCHAIN}/${triple}.cmake`];
+                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${linuxToolchainFile(target)}`];
                 }
                 break;
             }
             case 'win32': {
                 [dProgram, ...dParams] = params;
-                const triple = target.arch === 'x64' ? 'x86_64-w64-mingw32' : 'aarch64-w64-mingw32';
-                const tool = (name) => `${LLVM_MINGW}/bin/${triple}-${name}`;
-                platformParams = [
-                    '-e', `CC=${tool('clang')}`, '-e', `CXX=${tool('clang++')}`,
-                    '-e', `AR=${tool('ar')}`, '-e', `RANLIB=${tool('ranlib')}`,
-                    '-e', `NM=${tool('nm')}`, '-e', `STRIP=${tool('strip')}`,
-                    '-e', `RC=${tool('windres')}`, '-e', `WINDRES=${tool('windres')}`,
-                    '-e', 'PKG_CONFIG_LIBDIR=',
-                ];
+                platformParams = envParams(windowsBuildEnv(target));
                 if (dProgram === 'cmake' && dParams[0] !== '--build' && dParams[0] !== '--install') {
-                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${WINDOWS_TOOLCHAIN}/${triple}.cmake`];
+                    dParams = [...dParams, `-DCMAKE_TOOLCHAIN_FILE=${windowsToolchainFile(target)}`];
                 }
                 break;
             }
             case 'darwin': {
                 [dProgram, ...dParams] = params;
                 const appleArch = target.arch === 'x64' ? 'x86_64' : 'arm64';
-                // Homebrew and MacPorts packages exist on the build machine only: an archive compiled
-                // against one fails to link, or to load, anywhere else. The SDK's libraries stay visible.
+                // None of the machine's own packages (DARWIN_HOST_PACKAGE_PREFIXES); the SDK's libraries stay visible.
                 platformParams = ['-e', 'PKG_CONFIG_LIBDIR='];
                 if (dProgram === 'cmake') {
                     if (dParams[0] !== '--build' && dParams[0] !== '--install') {
@@ -309,7 +303,7 @@ export default function run(program, params = [], platformPrefix = null, target
                     // A --host triple sends configure after prefixed compilers (aarch64-apple-darwin-cc) that
                     // Xcode does not ship, and SQLite's autosetup has no fallback to the plain names.
                     platformParams.push(
-                        '-e', 'CC=/usr/bin/clang', '-e', 'CXX=/usr/bin/clang++',
+                        '-e', `CC=${DARWIN_CC}`, '-e', `CXX=${DARWIN_CXX}`,
                         '-e', `CFLAGS=${flags}`, '-e', `CXXFLAGS=${flags}`, '-e', `LDFLAGS=${flags}`,
                     );
                 }
@@ -320,14 +314,6 @@ export default function run(program, params = [], platformPrefix = null, target
     }
 
     const env = {};
-    let runner = 'DOCKER';
-    if (
-        ((HOST_BUILT_PLATFORMS.includes(target?.platform) || (target?.platform === 'wasi' && wasiHostSdk)) && program === null) ||
-        state.config.system.RUNNER === 'LOCAL'
-    ) {
-        runner = 'LOCAL';
-    }
-
     if (runner === 'LOCAL') {
         const allowedEnv = [
             '^PWD$',
@@ -366,6 +352,7 @@ export default function run(program, params = [], platformPrefix = null, target
         }
     }
 
+    const containerOptions = { cwd: buildPath, stdio: dockerOptions.console ? 'inherit' : ['ignore', 'pipe', 'pipe'], maxBuffer: EXEC_MAX_BUFFER };
     let fileExecParams;
     if (runner === 'LOCAL') {
         env.PATH = `/opt/homebrew/bin:${env.PATH}`;
@@ -383,17 +370,26 @@ export default function run(program, params = [], platformPrefix = null, target
             maxBuffer: EXEC_MAX_BUFFER,
         };
         fileExecParams = [dProgram, dParams, options];
-    } else if (runner === 'DOCKER') {
+    } else if (runner === 'REMOTE') {
+        fileExecParams = remoteExecParams({
+            remote,
+            mounts: [baseMount(state.config, { extraInputs: referencedPaths(state.config.paths.base, [dProgram, ...dParams, ...Object.values(env)]) })],
+            role,
+            image: getDockerImage(role, target?.platform === 'android' ? 'linux/amd64' : undefined),
+            cwd: replaceBasePathForDocker(dockerOptions.workdir || buildPath),
+            argv: [replaceBasePathForDocker(dProgram), ...replaceBasePathForDocker(dParams)],
+            env: replaceBasePathForDocker(env),
+        }, containerOptions);
+    } else {
         const dockerEnv = [];
         Object.entries(env).forEach(([key, value]) => {
             dockerEnv.push('-e', `${key}=${value}`);
         });
-        const options = { cwd: buildPath, stdio: dockerOptions.console ? 'inherit' : ['ignore', 'pipe', 'pipe'], maxBuffer: EXEC_MAX_BUFFER };
+        const options = containerOptions;
 
         let runnerParams;
         let imageOrContainer;
-        const role = imageRoleFor(target);
-        if (state.config.system.RUNNER === 'DOCKER_RUN') {
+        if (runner === 'DOCKER_RUN') {
             imageOrContainer = getDockerImage(role);
             runnerParams = ['run', '--rm', ...DOCKER_RUN_SECURITY_ARGS, '-v', `${state.config.paths.base}:/tmp/crossbind/live`];
             if (target?.platform === 'android') {
@@ -402,11 +398,9 @@ export default function run(program, params = [], platformPrefix = null, target
                 imageOrContainer = getDockerImage(role, 'linux/amd64');
                 runnerParams.push('--platform', 'linux/amd64');
             }
-        } else if (state.config.system.RUNNER === 'DOCKER_EXEC') {
+        } else {
             imageOrContainer = getDockerContainerName(state.config.paths.base, role);
             runnerParams = ['exec'];
-        } else {
-            throw new Error(`The runner ${state.config.system.RUNNER} is invalid.`);
         }
 
         const args = [
@@ -422,12 +416,15 @@ export default function run(program, params = [], platformPrefix = null, target
             ...replaceBasePathForDocker(dParams),
         ];
         fileExecParams = ['docker', args, options];
-    } else {
-        throw new Error(`The runner ${state.config.system.RUNNER} or command is invalid.`);
     }
 
     try {
-        execFileSync(...fileExecParams);
+        // A remote step takes no share of the local Docker VM, and its runner runs one step at a time.
+        if (isDocker && dockerOptions.exclusive) {
+            withDirLockSync(DOCKER_COMPILE_LOCK, () => execFileSync(...fileExecParams), { heldBy: 'another build is compiling in Docker' });
+        } else {
+            execFileSync(...fileExecParams);
+        }
     } catch (e) {
         if (e?.stdout?.length) console.log(e.stdout.toString());
         if (e?.stderr?.length) console.error(e.stderr.toString());