crossbind 2.0.0-beta.60 → 2.0.0-beta.63

package/src/utils/runConan.js

@@ -0,0 +1,257 @@
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import getOsUserAndGroupId from './getOsUserAndGroupId.js';
+import makeTreeWritable from './makeTreeWritable.js';
+import pullDockerImage, { getDockerImage, getDockerContainerName, imageRoleFor } from './pullDockerImage.js';
+import { DOCKER_RUN_SECURITY_ARGS } from './dockerSecurity.js';
+import assertExecContainer from './execContainer.js';
+import { IOS_DEVELOPER_DIR, XCODE_TOOLCHAIN_BIN } from './iosToolchain.js';
+import { DARWIN_CC, DARWIN_TOOLS_BIN } from './darwinToolchain.js';
+import { HOST_BUILT_PLATFORMS } from './targets.js';
+import { remoteExecParams } from './remoteRunner.js';
+import { runnerFor } from './selectRunner.js';
+
+// Every conan invocation crossbind makes goes through here. The config is passed in instead of read
+// from state, because state attaches the staged Conan packages while it is still being built.
+//
+// A Conan home holds code conan runs (plugins, hooks) and settings every recipe obeys (global.conf,
+// remotes), and the recipes it builds can write to it. So every run gets a home of its own, made in a
+// fresh work directory and deleted with it; only the package store outlives a run. On the host the
+// environment is rebuilt from an allowlist, so CONAN_DEFAULT_PROFILE, a stray CC or a remote's
+// credentials never reach conan.
+const ALLOWED_ENV = [
+    /^PATH$/,
+    /^HOME$/,
+    /^USER$/,
+    /^LOGNAME$/,
+    /^SHELL$/,
+    /^TERM$/,
+    /^TMPDIR$/,
+    /^TEMP$/,
+    /^TMP$/,
+    /^LANG$/,
+    /^LC_[A-Z_]+$/,
+    /^(HTTP|HTTPS|ALL|NO)_PROXY$/i,
+    /^SSL_CERT_(FILE|DIR)$/,
+    /^REQUESTS_CA_BUNDLE$/,
+    // A host Emscripten under RUNNER=LOCAL.
+    /^EMSDK(_NODE|_PYTHON)?$/,
+    /^EM_(CONFIG|CACHE)$/,
+    /^SYSTEMROOT$/i,
+    /^WINDIR$/i,
+    /^USERPROFILE$/i,
+    /^PROGRAMDATA$/i,
+];
+
+// What conan can build on the host: wasm with a host Emscripten, iOS and macOS with Xcode.
+const HOST_PLATFORMS = ['wasm', 'ios', 'darwin'];
+const APPLE_PLATFORM_NAMES = { ios: 'iOS', darwin: 'macOS' };
+// Where a container sees conanRoot(): an exec container mounts all of it here, a run container only
+// the store and its own work directory, at the same places.
+const CONTAINER_ROOT = '/var/cache/crossbind/conan';
+// The package folders conan reports, which the host stages from: downloaded binaries sit in p/<ref>/p, ones
+// conan built in b/<ref>/p. Sources and build folders stay on a remote runner.
+const REMOTE_STORE_OUTPUTS = ['p/*/p', 'b/*/p'];
+// A recipe's build log is the whole compiler output of a library.
+const MAX_BUFFER = 256 * 1024 * 1024;
+// The JSON graph carries conandata, the source URL and SHA-256 of the license rows, from 2.19 on.
+const MIN_CONAN_VERSION = [2, 19];
+
+export function conanRunner(config, target) {
+    const { runner } = runnerFor(imageRoleFor(target), config.system);
+    // Xcode runs on the Mac alone, so iOS and macOS packages build there whatever runner the rest use.
+    return HOST_BUILT_PLATFORMS.includes(target?.platform) ? 'LOCAL' : runner;
+}
+
+// Machine-wide like the cargo cache: built packages are reused by every project, and `pnpm run clear`
+// in one project must not throw them away. Containers write to the store and conan runs the recipes
+// it keeps there, so conan on the host never shares a store with them.
+export function conanRoot(runner = 'DOCKER_RUN') {
+    return path.join(os.homedir(), '.crossbind', runner === 'LOCAL' ? 'conan-local' : 'conan');
+}
+
+// The directory one conan run reads its inputs from and writes its outputs to, with the home it runs
+// with. conanPath names a file of it as conan sees it.
+export function createConanWork(runner) {
+    const root = conanRoot(runner);
+    const store = path.join(root, 'store');
+    fs.mkdirSync(store, { recursive: true });
+    fs.mkdirSync(path.join(root, 'work'), { recursive: true });
+    const dir = fs.mkdtempSync(path.join(root, 'work', 'run-'));
+    const conanDir = runner === 'LOCAL' ? dir : `${CONTAINER_ROOT}/work/${path.basename(dir)}`;
+    const conanPath = (file) => (runner === 'LOCAL' ? path.join(dir, file) : `${conanDir}/${file}`);
+    // conan takes its home from the first .conanrc at or above its working directory, ahead of
+    // CONAN_HOME; one in the working directory itself is always that first one.
+    fs.writeFileSync(path.join(dir, '.conanrc'), 'conan_home=./home\n');
+    fs.mkdirSync(path.join(dir, 'home'));
+    fs.writeFileSync(path.join(dir, 'home', 'global.conf'), [
+        `core.cache:storage_path=${runner === 'LOCAL' ? store : `${CONTAINER_ROOT}/store`}`,
+        'core:non_interactive=True',
+        '',
+    ].join('\n'));
+    return {
+        runner, root, store, dir, conanDir, conanPath,
+    };
+}
+
+function removeTree(dir) {
+    if (!fs.existsSync(dir)) return;
+    makeTreeWritable(dir);
+    fs.rmSync(dir, { recursive: true, force: true });
+}
+
+export const removeConanWork = (work) => removeTree(work.dir);
+
+// What runs that were killed left behind. Only while holding the install lock: no run is going then.
+export const clearConanWork = (runner) => removeTree(path.join(conanRoot(runner), 'work'));
+
+// conan reports paths as it sees them. In a container that is through the mounts, which all sit under
+// CONTAINER_ROOT, so mapping the prefix back is exact; a path anywhere else is none crossbind gave it.
+export function toHostPath(reported, work) {
+    if (work.runner === 'LOCAL') return path.resolve(reported);
+    const clean = path.posix.normalize(reported);
+    // A backslash is a separator again once the path is joined on a Windows host.
+    if (reported.includes('\\') || !clean.startsWith(`${CONTAINER_ROOT}/`)) {
+        throw new Error(`crossbind: conan reported ${JSON.stringify(reported)}, outside the directories crossbind mounts for it.`);
+    }
+    return path.join(work.root, clean.slice(CONTAINER_ROOT.length + 1));
+}
+
+function allowedEnv() {
+    return Object.fromEntries(Object.entries(process.env).filter(([key]) => ALLOWED_ENV.some((allowed) => allowed.test(key))));
+}
+
+// The Xcode every crossbind iOS archive builds with, whichever one xcode-select points at.
+const iosEnv = () => ({ ...allowedEnv(), DEVELOPER_DIR: IOS_DEVELOPER_DIR });
+
+// A GNU ar or nm ahead of Apple's on a Mac's PATH (Homebrew's binutils) writes archives Apple's linker
+// cannot read, and recipes, the build tools they build and Meson all take these tools by name. So the
+// run gets links to Xcode's first on its PATH; for macOS to the ones xcode-select picks, as the macOS
+// ports are built with.
+const APPLE_TOOLS = {
+    ar: 'ar', as: 'as', nm: 'llvm-nm', ranlib: 'ranlib', strip: 'strip',
+};
+const appleToolOf = (target, name) => (target.platform === 'darwin'
+    ? path.join(DARWIN_TOOLS_BIN, name)
+    : path.join(XCODE_TOOLCHAIN_BIN, APPLE_TOOLS[name]));
+
+function withAppleTools(work, env, target) {
+    if (process.platform !== 'darwin') return env;
+    const dir = path.join(work.dir, 'apple-tools');
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir);
+        Object.keys(APPLE_TOOLS).forEach((name) => fs.symlinkSync(appleToolOf(target, name), path.join(dir, name)));
+    }
+    return { ...env, PATH: [dir, env.PATH].filter(Boolean).join(path.delimiter) };
+}
+
+const toolVersions = new Map();
+// The first line a tool prints for --version, asked once.
+function toolVersion(command, env = allowedEnv(), cwd = undefined) {
+    const key = command.join(' ');
+    if (!toolVersions.has(key)) {
+        const [program, ...args] = command;
+        const result = spawnSync(program, [...args, '--version'], { encoding: 'utf8', env, cwd });
+        toolVersions.set(key, `${result.stdout ?? ''}`.split('\n')[0].trim());
+    }
+    return toolVersions.get(key);
+}
+
+// conan migrates the home it starts in, even for --version, so it is asked from a home of crossbind's
+// instead of the user's ~/.conan2.
+function conanVersion() {
+    const dir = path.join(conanRoot('LOCAL'), 'probe');
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(path.join(dir, '.conanrc'), 'conan_home=./home\n');
+    return toolVersion(['conan'], allowedEnv(), dir);
+}
+
+function compilerIdentity(target) {
+    if (target?.platform === 'ios') return toolVersion(['xcrun', 'clang'], iosEnv());
+    if (target?.platform === 'darwin') return toolVersion([DARWIN_CC]);
+    return toolVersion(['emcc']);
+}
+
+// What a host conan builds with; for a container the image digest says the same.
+export function localToolchainIdentity(target) {
+    return `${compilerIdentity(target)}\n${conanVersion()}`;
+}
+
+function assertLocalConan(target) {
+    const conan = conanVersion();
+    const [major, minor] = (conan.match(/(\d+)\.(\d+)/) ?? []).slice(1).map(Number);
+    const [needMajor, needMinor] = MIN_CONAN_VERSION;
+    if (major > needMajor || (major === needMajor && minor >= needMinor)) return;
+    const need = `Conan ${MIN_CONAN_VERSION.join('.')} or later; found ${conan || 'none'}.`;
+    throw new Error(HOST_BUILT_PLATFORMS.includes(target.platform)
+        ? `crossbind: Conan packages for ${APPLE_PLATFORM_NAMES[target.platform]} build on this Mac, with the conan on the PATH, which must be ${need} Install or upgrade it with \`brew install conan\` (\`brew upgrade conan\`) or \`pipx install conan\`.`
+        : `crossbind: RUNNER=LOCAL runs conan from the PATH and needs ${need}`);
+}
+
+export default function runConan(args, { config, target, work }) {
+    const options = { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf8', maxBuffer: MAX_BUFFER };
+    const role = imageRoleFor(target);
+    if (work.runner === 'LOCAL') {
+        if (!HOST_PLATFORMS.includes(target.platform)) {
+            throw new Error(`crossbind: Conan packages for ${target.platform} build in the ${role} image, which RUNNER=LOCAL does not use.`);
+        }
+        if (HOST_BUILT_PLATFORMS.includes(target.platform) && process.platform !== 'darwin') {
+            throw new Error(`crossbind: Conan packages for ${APPLE_PLATFORM_NAMES[target.platform]} build with Xcode, on a Mac.`);
+        }
+        assertLocalConan(target);
+        const env = withAppleTools(work, { ...(target.platform === 'ios' ? iosEnv() : allowedEnv()), CONAN_HOME: path.join(work.dir, 'home') }, target);
+        return spawnSync('conan', args, { ...options, cwd: work.dir, env });
+    }
+
+    // Google ships the linux NDK for x86_64 only.
+    const platform = target.platform === 'android' ? 'linux/amd64' : undefined;
+    if (work.runner === 'REMOTE') {
+        return spawnSync(...remoteExecParams({
+            remote: runnerFor(role, config.system).remote,
+            role,
+            image: getDockerImage(role, platform),
+            mounts: [
+                { host: work.store, container: `${CONTAINER_ROOT}/store`, inputRoots: [], outputRoots: REMOTE_STORE_OUTPUTS },
+                { host: work.dir, container: work.conanDir, inputRoots: ['.'], outputRoots: ['.'] },
+            ],
+            cwd: work.conanDir,
+            argv: ['conan', ...args],
+            env: { CONAN_HOME: work.conanPath('home') },
+        }, options));
+    }
+
+    const env = ['-e', `CONAN_HOME=${work.conanPath('home')}`];
+    let runnerArgs;
+    if (work.runner === 'DOCKER_EXEC') {
+        const name = getDockerContainerName(config.paths.base, role);
+        assertExecContainer({
+            name,
+            mounts: [[CONTAINER_ROOT, work.root]],
+            workdir: work.conanDir,
+            hint: `Recreate it with \`crossbind docker stop ${role}\`, \`crossbind docker delete ${role}\` and \`crossbind docker create ${role}\`, which mount ${work.root} at ${CONTAINER_ROOT}.`,
+            mountNote: 'It predates the conan mount.',
+        });
+        runnerArgs = ['exec', ...env, '--user', getOsUserAndGroupId(), '--workdir', work.conanDir, name];
+    } else {
+        pullDockerImage(role, platform);
+        runnerArgs = [
+            'run',
+            '--rm',
+            ...(platform ? ['--platform', platform] : []),
+            ...DOCKER_RUN_SECURITY_ARGS,
+            '-v',
+            `${work.store}:${CONTAINER_ROOT}/store`,
+            '-v',
+            `${work.dir}:${work.conanDir}`,
+            ...env,
+            '--user',
+            getOsUserAndGroupId(),
+            '--workdir',
+            work.conanDir,
+            getDockerImage(role, platform),
+        ];
+    }
+    return spawnSync('docker', [...runnerArgs, 'conan', ...args], options);
+}

package/src/utils/runtimeEntries.js

@@ -0,0 +1,56 @@
+// The one module a package publishes per runtime binary, dist/<runtimeEnv>/<format>.mjs. It exports every
+// bound name as an ESM live binding that initNative() fills, the way a bundler's module for a header does,
+// so `import { initNative, X } from '<package>/node/napi'` works with or without init options.
+
+const PROXY_NAMES = new Set(['initNative', 'AllSymbols']);
+
+// A name is a plain string or { local, wire }: the second exports `local` while reading the differently
+// registered `wire` off the module. Two headers may bind one name; the module exports it once.
+export function uniqueBindings(names) {
+    const seen = new Set();
+    return names
+        .map((name) => (typeof name === 'string' ? { local: name, wire: name } : name))
+        .filter(({ local }) => {
+            if (PROXY_NAMES.has(local) || seen.has(local)) return false;
+            seen.add(local);
+            return true;
+        });
+}
+
+// Each runtime's loader, imported as `boot`, and the config it boots with; the binaries sit in dist, one
+// directory above the entry.
+export const ENTRY_RUNTIMES = {
+    // The Node-API loader and the Node wasm build are both CommonJS.
+    node: (loader) => ({
+        imports: ["import { createRequire } from 'node:module';", `const boot = createRequire(import.meta.url)('../${loader}');`],
+        config: 'config',
+    }),
+    // Edge runtimes cannot compile WebAssembly from bytes, so the binary is a module import their bundler resolves.
+    edge: (loader, wasm) => ({
+        imports: [`import boot from '../${loader}';`, `import wasm from '../${wasm}';`],
+        config: '{ getWasmFunction: () => wasm, ...config }',
+    }),
+};
+
+// A bound name may be a JavaScript reserved word, which an export list accepts but a declaration does not.
+export function runtimeEntryModule(names, { imports, config }) {
+    const bindings = uniqueBindings(names);
+    return [
+        '// Generated by crossbind - do not edit.',
+        ...imports,
+        '',
+        'export let AllSymbols = {};',
+        ...bindings.map(({ local }) => `let $${local} = null;`),
+        ...(bindings.length ? [`export { ${bindings.map(({ local }) => `$${local} as ${local}`).join(', ')} };`] : []),
+        '',
+        'export function initNative(config = {}) {',
+        `    return boot(${config}).then((m) => {`,
+        '        AllSymbols = m;',
+        ...bindings.map(({ local, wire }) => `        $${local} = m.${wire};`),
+        '        return m;',
+        '    });',
+        '}',
+        'initNative.terminate = () => boot.terminate();',
+        '',
+    ].join('\n');
+}