crossbind 2.0.0-beta.62 → 2.0.0-beta.63

package/src/utils/pthreadMainScript.js

@@ -0,0 +1,14 @@
+// The pthread bootstrap spawns its workers from _scriptName, which the glue captures from document.currentScript
+// when it loads: undefined for a bridge loaded as a module script, so the workers fetch "/undefined" and a
+// direct-mode mt init hangs. The spawn reads Module.crossbindMainScript first, which the browser adapter sets from
+// paths.worker/paths.js (emscripten's own mainScriptUrlOrBlob trips emsdk 6 debug assertions on st targets).
+
+// The release glue spells it `pthreadMainJs=_scriptName`, the debug glue `pthreadMainJs = _scriptName`.
+const SPAWN = /pthreadMainJs\s*=\s*_scriptName/g;
+const MAIN_SCRIPT = 'pthreadMainJs=Module["crossbindMainScript"]||_scriptName';
+
+// Returns the rewritten glue and whether it spawns pthreads from a script this rewrite does not recognise.
+export function spawnPthreadsFromMainScript(glue) {
+    const text = glue.replace(SPAWN, MAIN_SCRIPT);
+    return { text, missed: text.includes('pthreadMainJs') && !text.includes(MAIN_SCRIPT) };
+}

package/src/utils/pullDockerImage.js

@@ -20,9 +20,11 @@ const IMAGES = Object.fromEntries(
     ]),
 );
 
-// Which image a target compiles in: wasm and wasi share the web image, ios and darwin never reach docker.
+// Which image a target compiles in: wasm and wasi share the web image, linux and linuxmusl the linux
+// image, ios and darwin never reach docker.
 export function imageRoleFor(target) {
     if (target?.platform === 'android' || target?.platform === 'linux') return target.platform;
+    if (target?.platform === 'linuxmusl') return 'linux';
     if (target?.platform === 'win32') return 'windows';
     return 'web';
 }

package/src/utils/relocatePrebuilt.js

@@ -0,0 +1,119 @@
+import fs from 'node:fs';
+import upath from 'upath';
+
+// pkg-config files, libtool archives, CMake configs and -config scripts record absolute paths of
+// the machine that built them, so a published prebuilt only resolves there. These rewrites make
+// every path relative to where the tree lands; whatever they cannot rewrite is left for the
+// publish gate to report. Some upstream templates also name libstdc++, which no crossbind
+// toolchain links: every one of them builds against libc++.
+
+const PREFIX_VARIABLE = 'crossbind_prefix';
+const ASSIGNMENT = /^(\s*[A-Za-z_][A-Za-z0-9_]*=)(.*)$/;
+const SINGLE_QUOTED = /^'([^']*)'$/;
+const DOUBLE_QUOTED = /^"((?:[^"\\]|\\.)*)"$/;
+const BARE_WORD = /^[^\s'"\\]*$/;
+const EXPANDS_IN_DOUBLE_QUOTES = /[$`"\\]/;
+const PERMISSION_BITS = 0o7777;
+const OWNER_WRITE = 0o200;
+const LIBSTDCXX_FLAG = /(^|[\s"'=])-lstdc\+\+(?=[\s"']|$)/gm;
+
+const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+const nameLibcxx = (text) => text.replace(LIBSTDCXX_FLAG, '$1-lc++');
+
+const replaceEach = (text, needles, replacement) => needles
+    .reduce((result, needle) => result.replaceAll(needle, () => replacement), text);
+
+// CMake installs programs without the owner write bit, so the rewrite borrows it and gives it back.
+function rewrite(file, before, after) {
+    if (before === after) return;
+    const mode = fs.statSync(file).mode & PERMISSION_BITS;
+    fs.chmodSync(file, mode | OWNER_WRITE);
+    fs.writeFileSync(file, after);
+    fs.chmodSync(file, mode);
+}
+
+function filesUnder(dir) {
+    return fs.readdirSync(dir, { recursive: true, withFileTypes: true })
+        .filter((entry) => entry.isFile())
+        .map((entry) => upath.join(entry.parentPath, entry.name));
+}
+
+const isUnder = (path, prefixes) => prefixes.some((prefix) => path.startsWith(prefix));
+
+// Another package's files are only known by name once the tree leaves the build machine. Its
+// archive may sit in a lib directory or, from an xcframework, in a slice directory.
+function nameForeignArchives(text, { installPrefixes, buildBases }, spell) {
+    return buildBases.reduce((result, base) => result.replace(
+        new RegExp(`${escapeRegExp(base)}/[^\\s'";]*?/lib([\\w+.-]+)\\.a`, 'g'),
+        (archive, name) => (isUnder(archive, installPrefixes) ? archive : spell(name)),
+    ), text);
+}
+
+// Search paths into other packages, and object files of the build tree such as the WASI runtime
+// stubs a configure build links through LIBS.
+function dropForeignPaths(text, { installPrefixes, buildBases }) {
+    return buildBases.reduce((result, base) => result
+        .replace(
+            new RegExp(`\\s-[LI]${escapeRegExp(base)}/[^\\s'";]*`, 'g'),
+            (flag) => (isUnder(flag.trim().slice(2), installPrefixes) ? flag : ''),
+        )
+        .replace(
+            new RegExp(`\\s${escapeRegExp(base)}/[^\\s'";]*\\.o(?=[\\s'";]|$)`, 'gm'),
+            (object) => (isUnder(object.trim(), installPrefixes) ? object : ''),
+        ), text);
+}
+
+function relocatePkgConfig(file, prefixDir, paths) {
+    const toPrefix = upath.relative(upath.dirname(file), prefixDir);
+    const before = fs.readFileSync(file, 'utf8');
+    const own = replaceEach(before, paths.installPrefixes, `\${pcfiledir}/${toPrefix}`);
+    const foreign = nameForeignArchives(dropForeignPaths(own, paths), paths, (name) => `-l${name}`);
+    rewrite(file, before, nameLibcxx(foreign));
+}
+
+function relocateCMake(file, paths) {
+    const before = fs.readFileSync(file, 'utf8');
+    rewrite(file, before, nameForeignArchives(before, paths, (name) => name));
+}
+
+// The text of an assignment value as it reads between double quotes, or null when its quoting
+// is anything but one plain word.
+function doubleQuotedContent(value) {
+    const single = SINGLE_QUOTED.exec(value);
+    if (single) return EXPANDS_IN_DOUBLE_QUOTES.test(single[1]) ? null : single[1];
+    const double = DOUBLE_QUOTED.exec(value);
+    if (double) return double[1];
+    return BARE_WORD.test(value) ? value : null;
+}
+
+function relocateAssignment(line, installPrefixes) {
+    const match = ASSIGNMENT.exec(line);
+    if (!match || !installPrefixes.some((prefix) => line.includes(prefix))) return line;
+    const content = doubleQuotedContent(match[2]);
+    if (content === null) return line;
+    return `${match[1]}"${replaceEach(content, installPrefixes, `\${${PREFIX_VARIABLE}}`)}"`;
+}
+
+function relocateConfigScript(file, prefixDir, installPrefixes) {
+    const before = fs.readFileSync(file, 'utf8');
+    const lines = before.split('\n');
+    if (!/^#!.*sh/.test(lines[0])) return;
+    const relocated = lines.map((line) => relocateAssignment(line, installPrefixes));
+    const isPrefixRelocated = relocated.some((line, index) => line !== lines[index]);
+    const toPrefix = upath.relative(upath.dirname(file), prefixDir);
+    const definition = `${PREFIX_VARIABLE}=$(cd "$(dirname "$0")/${toPrefix}" && pwd)`;
+    const after = isPrefixRelocated ? [relocated[0], definition, ...relocated.slice(1)] : relocated;
+    rewrite(file, before, nameLibcxx(after.join('\n')));
+}
+
+const isConfigScript = (file) => file.endsWith('-config') && upath.basename(upath.dirname(file)) === 'bin';
+
+export default function relocatePrebuilt(prefixDir, paths) {
+    filesUnder(prefixDir).forEach((file) => {
+        if (file.endsWith('.la')) fs.rmSync(file);
+        else if (file.endsWith('.pc')) relocatePkgConfig(file, prefixDir, paths);
+        else if (file.endsWith('.cmake')) relocateCMake(file, paths);
+        else if (isConfigScript(file)) relocateConfigScript(file, prefixDir, paths.installPrefixes);
+    });
+}

package/src/utils/remoteClient.js

@@ -0,0 +1,127 @@
+// One toolchain step on the remote runner: send what changed below each mount's input roots, stream the
+// step's output, write the files it produced back through the mounts, exit with its exit code.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { walk, hashIndex, presentUnits } from '../runner/files.js';
+import { isHash } from '../runner/blobs.js';
+import { getContentHash } from './hash.js';
+import { outputTarget, writeOutputFile, removeOutputFile } from './remoteOutputs.js';
+import { request, expectOk, uploadMissing } from './remoteUpload.js';
+
+const INDEX_DIR = path.join(os.homedir(), '.crossbind', 'remote-index');
+
+const indexFileOf = (mount) => path.join(INDEX_DIR, `${getContentHash(`${mount.host}\n${mount.container}`).slice(0, 16)}.json`);
+
+function loadIndex(mount) {
+    try {
+        return new Map(Object.entries(JSON.parse(fs.readFileSync(indexFileOf(mount), 'utf8'))));
+    } catch {
+        return new Map();
+    }
+}
+
+function saveIndex(mount, index) {
+    fs.mkdirSync(INDEX_DIR, { recursive: true });
+    fs.writeFileSync(indexFileOf(mount), JSON.stringify(Object.fromEntries(index)));
+}
+
+function describeMount(mount, rules, index) {
+    const { files, dirs } = walk(mount.host, mount.inputRoots, rules);
+    const hashes = hashIndex(mount.host, files, index);
+    const manifest = Object.fromEntries([...hashes].map(([rel, sha256]) => [rel, { sha256, mode: files.get(rel).mode }]));
+    const sources = [...hashes].map(([rel, sha256]) => [sha256, path.join(mount.host, rel)]);
+    return { manifest, dirs, sources };
+}
+
+function writeOutput(target, entry, data, indexes) {
+    if (getContentHash(data) !== entry.sha256) throw new Error(`crossbind: ${target.file} arrived from the remote runner with the wrong content.`);
+    writeOutputFile(target, data, entry.mode);
+    const { size, mtimeMs } = fs.statSync(target.file);
+    indexes.get(target.mount).set(target.rel, { size, mtimeMs, sha256: entry.sha256 });
+}
+
+async function runStep(payload, described, present, indexes, received) {
+    const body = {
+        role: payload.role,
+        image: payload.image,
+        rules: payload.rules,
+        cwd: payload.cwd,
+        argv: payload.argv,
+        env: payload.env,
+        mounts: payload.mounts.map((mount, i) => ({
+            path: mount.container,
+            roots: mount.inputRoots,
+            outputRoots: mount.outputRoots,
+            manifest: described[i].manifest,
+            dirs: described[i].dirs,
+            present: present[i],
+        })),
+    };
+    const response = await request(payload.url, '/v1/exec', { method: 'POST', body: JSON.stringify(body) });
+    if (response.status !== 200) await expectOk(response, 'to run the step');
+
+    const decoder = new TextDecoder();
+    let pending = '';
+    let result = null;
+    const handle = (record) => {
+        if ('exit' in record) result = record;
+        if (record.stdout) process.stdout.write(record.stdout);
+        if (record.stderr) process.stderr.write(record.stderr);
+        if (record.file) {
+            writeOutput(outputTarget(payload.mounts, present, record.file), record, Buffer.from(record.data, 'base64'), indexes);
+            received.add(record.file);
+        }
+    };
+    for await (const chunk of response.body) {
+        pending += decoder.decode(chunk, { stream: true });
+        let newline = pending.indexOf('\n');
+        while (newline !== -1) {
+            if (newline > 0) handle(JSON.parse(pending.slice(0, newline)));
+            pending = pending.slice(newline + 1);
+            newline = pending.indexOf('\n');
+        }
+    }
+    if (!result) throw new Error('crossbind: the remote runner closed the connection before the step finished.');
+    if (result.error) throw new Error(`crossbind: the remote runner failed during the step: ${result.error}.`);
+    return result;
+}
+
+// Outputs too large for the stream are fetched by hash.
+async function writeOutputs(payload, result, present, indexes, received) {
+    for (const [file, entry] of Object.entries(result.outputs).filter(([name]) => !received.has(name))) {
+        const target = outputTarget(payload.mounts, present, file);
+        if (!isHash(entry.sha256)) throw new Error(`crossbind: the remote runner named ${file} without a sha256 digest.`);
+        const response = await request(payload.url, `/v1/blobs/${entry.sha256}`);
+        if (response.status !== 200) throw new Error(`crossbind: could not download ${file} from the remote runner (${response.status}).`);
+        writeOutput(target, entry, Buffer.from(await response.arrayBuffer()), indexes);
+    }
+    result.removed.forEach((file) => {
+        const target = outputTarget(payload.mounts, present, file);
+        removeOutputFile(target);
+        indexes.get(target.mount).delete(target.rel);
+    });
+}
+
+async function main() {
+    const payload = JSON.parse(process.argv[2]);
+    const indexes = new Map(payload.mounts.map((mount) => [mount, loadIndex(mount)]));
+    const described = payload.mounts.map((mount) => describeMount(mount, payload.rules, indexes.get(mount)));
+    await uploadMissing(payload.url, new Map(described.flatMap((d) => d.sources)));
+
+    // The store units this machine holds before the step: the runner sends only the others, and may write only those.
+    const present = payload.mounts.map((mount) => presentUnits(mount.host, mount.outputRoots));
+    const received = new Set();
+    const result = await runStep(payload, described, present, indexes, received);
+    await writeOutputs(payload, result, present, indexes, received);
+    indexes.forEach((index, mount) => saveIndex(mount, index));
+    return result.exit;
+}
+
+main().then(
+    (code) => process.exit(code),
+    (error) => {
+        process.stderr.write(`${error.message}\n`);
+        process.exit(1);
+    },
+);

package/src/utils/remoteOutputs.js

@@ -0,0 +1,39 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { hostPathOf, isDeclaredOutput, modeOf } from '../runner/files.js';
+
+// Where a file the runner hands back may land: below an output root the step declared, and in a store
+// unit only when this machine does not hold it yet. `present` lists those units per mount.
+export function outputTarget(mounts, present, containerPath) {
+    const target = hostPathOf(mounts, containerPath);
+    if (!isDeclaredOutput(target.rel, target.mount.outputRoots, present[mounts.indexOf(target.mount)])) {
+        throw new Error(`crossbind: the remote runner returned ${containerPath}, which is not an output of this step.`);
+    }
+    return target;
+}
+
+// A link inside the project must not carry a write or a delete out of it.
+function assertStaysInside(target) {
+    fs.mkdirSync(target.mount.host, { recursive: true });
+    const root = fs.realpathSync(target.mount.host);
+    let dir = path.dirname(target.file);
+    while (!fs.existsSync(dir)) dir = path.dirname(dir);
+    const real = fs.realpathSync(dir);
+    if (real !== root && !real.startsWith(`${root}${path.sep}`)) {
+        throw new Error(`crossbind: ${target.file} leads out of ${target.mount.host} through a link, so the remote runner's output stays unwritten.`);
+    }
+}
+
+export function writeOutputFile(target, data, mode) {
+    assertStaysInside(target);
+    fs.mkdirSync(path.dirname(target.file), { recursive: true });
+    // A link at the output's own path is replaced, never written through.
+    if (fs.lstatSync(target.file, { throwIfNoEntry: false })?.isSymbolicLink()) fs.rmSync(target.file);
+    fs.writeFileSync(target.file, data);
+    fs.chmodSync(target.file, modeOf(mode));
+}
+
+export function removeOutputFile(target) {
+    assertStaysInside(target);
+    fs.rmSync(target.file, { force: true });
+}

package/src/utils/remoteRunner.js

@@ -0,0 +1,97 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DOCKER_BASE } from './replaceBasePathForDocker.js';
+import { remoteVariables } from './selectRunner.js';
+
+const CLIENT = fileURLToPath(new URL('./remoteClient.js', import.meta.url));
+
+// Only native inputs travel to the runner; the application's JavaScript and credential files stay on the
+// machine. A cargo target folder stays on the runner, and only the static libraries crossbind links come back from it.
+export const REMOTE_EXCLUDE_RULES = Object.freeze({
+    dirs: Object.freeze(['node_modules', '.git']),
+    files: Object.freeze(['.env', '.env.*', '.npmrc', '.yarnrc', '.yarnrc.yml', '.netrc']),
+    extensions: Object.freeze(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']),
+    serverOnly: Object.freeze({ names: Object.freeze(['target', 'target-mt']), marker: 'Cargo.toml', keep: Object.freeze(['*/release/*.a']) }),
+});
+
+const TOKEN_VARIABLE = 'CROSSBIND_TOKEN';
+
+const LOOPBACK = /^(localhost|127(\.\d{1,3}){3}|\[::1\])$/;
+const warnedUrls = new Set();
+
+// Plain http to another machine carries the token and the sources unencrypted.
+function warnIfPlainHttp(url) {
+    const { protocol, hostname } = new URL(url);
+    if (protocol !== 'http:' || LOOPBACK.test(hostname) || warnedUrls.has(url)) return;
+    warnedUrls.add(url);
+    console.warn(`crossbind: ${url} is plain http to another machine - the token and your sources travel unencrypted; put TLS in front of the runner.`);
+}
+
+const isInside = (rel, root) => root === '.' || rel === root || rel.startsWith(`${root}/`);
+
+// Null for a folder outside the base: a local docker run mounts only the base, so it never sees one either.
+function relativeToBase(base, folder) {
+    const rel = path.relative(base, folder);
+    if (rel.startsWith('..') || path.isAbsolute(rel)) return null;
+    return rel.split(path.sep).join('/') || '.';
+}
+
+function foldNested(roots) {
+    const unique = [...new Set(roots)].sort((a, b) => a.length - b.length);
+    return unique.filter((root, i) => !unique.slice(0, i).some((outer) => isInside(root, outer)));
+}
+
+// A file the command line names may include its neighbours (`#include "../include/x.h"`), so unless a
+// configured folder holds it, the package that holds it travels: the nearest folder below the base with a
+// package.json. Its JavaScript stays behind, as everywhere else.
+function packageOf(base, configured, file) {
+    if (configured.some((root) => isInside(relativeToBase(base, file), root))) return file;
+    for (let dir = path.dirname(file); ![null, '.'].includes(relativeToBase(base, dir)); dir = path.dirname(dir)) {
+        if (fs.existsSync(path.join(dir, 'package.json'))) return dir;
+    }
+    return file;
+}
+
+// What a toolchain step may read (inputRoots) and where it writes (outputRoots), relative to paths.base,
+// the folder a local docker run mounts. `extraInputs` are only read (a file the link names);
+// `extraOutputs` are read and written, like a crate's folder and its Cargo.lock.
+export function remoteRoots(config, { extraInputs = [], extraOutputs = [] } = {}) {
+    const { paths, allDependencies = [] } = config;
+    const relOf = (folder) => relativeToBase(paths.base, folder);
+    const isBelowBase = (folder) => Boolean(folder) && relOf(folder) !== null;
+    const dependencyFolders = allDependencies.flatMap((d) => [d.paths.output, ...(d.paths.native ?? []), ...(d.paths.header ?? [])]);
+    const configured = [...paths.native, ...paths.header, paths.cache, paths.output, `${paths.cli}/assets`, ...dependencyFolders].filter(isBelowBase).map(relOf);
+    const outputs = [paths.cache, paths.output, ...extraOutputs].filter(isBelowBase).map(relOf);
+    return {
+        inputRoots: foldNested([...configured, ...extraInputs.filter(isBelowBase).map((file) => relOf(packageOf(paths.base, configured, file))), ...outputs]),
+        outputRoots: foldNested(outputs),
+    };
+}
+
+export function baseMount(config, extra = {}) {
+    return { host: config.paths.base, container: DOCKER_BASE, ...remoteRoots(config, extra) };
+}
+
+const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+// Paths a command line names below the base (a source the link adds, an include folder, a preload file)
+// travel even when no configured folder holds them. The base itself is never one: it would send everything.
+export function referencedPaths(base, values) {
+    const pattern = new RegExp(`${escapeRegExp(base)}(?:/[^@=;:,'"\\s]*)?`, 'g');
+    const found = values.flatMap((value) => String(value).match(pattern) ?? []);
+    return [...new Set(found)].filter((candidate) => candidate !== base && fs.existsSync(candidate));
+}
+
+// Arguments for execFileSync/spawnSync: the client runs as its own process, so the caller stays
+// synchronous as it is with docker. The token reaches it through its environment, never its command line.
+export function remoteExecParams({ remote, role, image, mounts, cwd, argv, env }, options) {
+    if (!remote) {
+        throw new Error(`crossbind: RUNNER=REMOTE, but the ${role} image has no runner address - set ${remoteVariables(role).key} (or REMOTE_URL), or build with CROSSBIND_RUNNER=DOCKER_RUN. A machine that names no runner address builds on crossbind cloud after crossbind login.`);
+    }
+    const { url, token, tokenVariable } = remote;
+    if (!token) throw new Error(`crossbind: the ${role} runner at ${url} needs a token - set ${tokenVariable}.`);
+    warnIfPlainHttp(url);
+    const payload = { url, role, image, mounts, cwd, argv, env, rules: REMOTE_EXCLUDE_RULES };
+    return [process.execPath, [CLIENT, JSON.stringify(payload)], { ...options, env: { ...process.env, [TOKEN_VARIABLE]: token } }];
+}

package/src/utils/remoteUpload.js

@@ -0,0 +1,134 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+// Raw bytes per upload request, a batch of small files or a part of a large one. With base64 a batch stays below the
+// 32 MiB a Cloud Run request may carry, and a part below Cloudflare's 100 MB as well.
+const UPLOAD_BATCH_BYTES = 16 * 1024 * 1024;
+// The first runner protocol that joins a file sent in parts.
+const PARTS_PROTOCOL = 2;
+const MB = 1024 * 1024;
+// A long upload to a hosted runner now and then loses its connection: a request goes again this many times, waiting a
+// second longer each time.
+const UPLOAD_RETRIES = 4;
+const RETRY_DELAY_MS = 1000;
+// Failures before a connection opened, such as a name lookup that failed for a moment: the request never reached the
+// runner, so it goes again this many times. After a connection opened it never does, since a step may have started.
+const UNSENT_CODES = new Set(['ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH', 'UND_ERR_CONNECT_TIMEOUT']);
+const UNSENT_RETRIES = 2;
+
+const wait = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); });
+
+export async function request(url, route, init = {}) {
+    const token = process.env.CROSSBIND_TOKEN;
+    const headers = { ...(token ? { authorization: `Bearer ${token}` } : {}), ...init.headers };
+    for (let attempt = 1; ; attempt += 1) {
+        try {
+            // A redirect would send the request, sources included, to an address nobody configured.
+            return await fetch(`${url}${route}`, { ...init, headers, redirect: 'error' });
+        } catch (error) {
+            if (!UNSENT_CODES.has(error.cause?.code) || attempt > UNSENT_RETRIES) {
+                throw new Error(`crossbind: the remote runner at ${url} is unreachable (${error.cause?.code ?? error.message}).`, { cause: error });
+            }
+            await wait(RETRY_DELAY_MS * attempt);
+        }
+    }
+}
+
+export async function expectOk(response, what) {
+    if (response.ok) return response.json();
+    const body = await response.json().catch(() => ({}));
+    throw new Error(`crossbind: the remote runner failed ${what}: ${body.error ?? response.status}.`);
+}
+
+async function requestWithRetries(url, route, init) {
+    for (let attempt = 1; ; attempt += 1) {
+        const response = await request(url, route, init).catch((error) => error);
+        const failed = response instanceof Error || response.status >= 500;
+        if (!failed || attempt > UPLOAD_RETRIES) {
+            if (response instanceof Error) throw response;
+            return response;
+        }
+        await wait(RETRY_DELAY_MS * attempt);
+    }
+}
+
+async function runnerProtocol(url) {
+    const health = await expectOk(await request(url, '/v1/health'), 'to answer its health check');
+    return health.protocol ?? 1;
+}
+
+// One request per part, in order; the runner checks the joined file against its hash. After a part whose connection
+// dropped, the upload goes on from where the runner says it stands. The upload is named after the file, so a later
+// attempt, or another client sending the same file, goes on with what the runner holds.
+async function uploadInParts(url, hash, file, total, partBytes) {
+    const upload = hash.slice(0, 32);
+    const fd = fs.openSync(file, 'r');
+    let resumes = 0;
+    try {
+        for (let start = 0; start < total;) {
+            const part = Buffer.alloc(Math.min(partBytes, total - start));
+            fs.readSync(fd, part, 0, part.length, start);
+            const response = await requestWithRetries(url, `/v1/blobs/${hash}`, {
+                method: 'PUT',
+                body: part,
+                headers: { 'content-range': `bytes ${start}-${start + part.length - 1}/${total}`, 'x-crossbind-upload': upload },
+            });
+            if (response.status === 413) {
+                throw new Error(`crossbind: the remote runner at ${url} took no part of ${path.basename(file)}: it refused a part of ${part.length} bytes, so the proxy in front of it takes less in one request.`);
+            }
+            if (response.status === 409 && resumes < UPLOAD_RETRIES) {
+                const { received } = await response.json();
+                if (!Number.isSafeInteger(received) || received < 0 || received >= total) throw new Error(`crossbind: the remote runner holds ${received} bytes of ${path.basename(file)}, which has ${total}.`);
+                resumes += 1;
+                start = received;
+                continue;
+            }
+            const answer = await expectOk(response, `to receive ${path.basename(file)}`);
+            if (response.status === 201) return;
+            start = answer.received;
+        }
+    } finally {
+        fs.closeSync(fd);
+    }
+}
+
+// A runner deployed before uploads in parts takes a large file in one request, which a body limit in front of it refuses.
+async function uploadWhole(url, hash, file, total) {
+    const response = await request(url, `/v1/blobs/${hash}`, { method: 'PUT', body: fs.readFileSync(file) });
+    if (response.status === 413) {
+        throw new Error(`crossbind: the remote runner at ${url} refused ${path.basename(file)} (${Math.ceil(total / MB)} MB) as too large for one request; deploy the runner again with this crossbind version, which sends large files in parts.`);
+    }
+    await expectOk(response, `to receive ${path.basename(file)}`);
+}
+
+// `sources` maps each hash to a file holding it; only what the runner lacks travels, small files in a few batches and a
+// large one on its own.
+export async function uploadMissing(url, sources, partBytes = UPLOAD_BATCH_BYTES) {
+    const listing = await request(url, '/v1/missing', { method: 'POST', body: JSON.stringify({ hashes: [...sources.keys()] }) });
+    const { missing } = await expectOk(listing, 'to list the files it lacks');
+    let protocol;
+    let batch = [];
+    let size = 0;
+    const flush = async () => {
+        if (batch.length === 0) return;
+        const body = batch.map(({ hash, data }) => JSON.stringify({ sha256: hash, data: data.toString('base64') })).join('\n');
+        await expectOk(await requestWithRetries(url, '/v1/blobs', { method: 'POST', body }), `to receive ${batch.length} files`);
+        batch = [];
+        size = 0;
+    };
+    for (const hash of missing) {
+        const file = sources.get(hash);
+        const fileSize = fs.statSync(file).size;
+        if (fileSize > partBytes) {
+            protocol ??= await runnerProtocol(url);
+            if (protocol >= PARTS_PROTOCOL) await uploadInParts(url, hash, file, fileSize, partBytes);
+            else await uploadWhole(url, hash, file, fileSize);
+            continue;
+        }
+        const data = fs.readFileSync(file);
+        if (size > 0 && size + data.length > partBytes) await flush();
+        batch.push({ hash, data });
+        size += data.length;
+    }
+    await flush();
+}

package/src/utils/replaceFile.js

@@ -0,0 +1,8 @@
+import fs from 'node:fs';
+
+// macOS keeps the code signature of a loaded binary per inode: a rebuilt addon or command copied over
+// one that already ran gets the next process that loads it killed. A new file gets a new inode.
+export default function replaceFile(source, destination) {
+    fs.rmSync(destination, { force: true });
+    fs.copyFileSync(source, destination);
+}