crossbind 2.0.0-beta.62 → 2.0.0-beta.64

package/src/runner/files.js

@@ -0,0 +1,193 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+// Shared by the client (utils/remoteClient.js) and the runner (server.js). The runner reaches the toolchain
+// image as this folder alone, mounted or copied in, so nothing here may import from outside it.
+
+export const sha256Of = (data) => crypto.createHash('sha256').update(data).digest('hex');
+
+// `name` against a file name pattern: a trailing `*` matches any rest (`.env.*`).
+const matchesName = (name, pattern) => (pattern.endsWith('*') ? name.startsWith(pattern.slice(0, -1)) : name === pattern);
+
+// `rel` is relative to the root being walked, so a root that itself sits in node_modules (an installed port) still counts.
+export function isExcluded(rel, rules) {
+    const segments = rel.split('/');
+    if (segments.some((segment) => rules.dirs.includes(segment))) return true;
+    if (rules.files?.some((pattern) => matchesName(segments.at(-1), pattern))) return true;
+    return rules.extensions.includes(path.posix.extname(rel));
+}
+
+// Segment by segment: `*` matches any one segment, `*.a` any segment with that ending.
+export function matchesPattern(rel, pattern) {
+    const parts = rel.split('/');
+    const wanted = pattern.split('/');
+    return parts.length === wanted.length && wanted.every((want, i) => want === '*'
+        || (want.startsWith('*.') ? parts[i].endsWith(want.slice(1)) : want === parts[i]));
+}
+
+// A build tree such as a cargo target folder stays on the runner: a sync walk skips it, so it is neither
+// uploaded nor deleted, and an output walk returns only the artifacts its `keep` patterns name.
+function isServerOnly(base, rel, rules) {
+    const serverOnly = rules.serverOnly;
+    if (!serverOnly?.names.includes(path.posix.basename(rel))) return false;
+    return fs.existsSync(path.join(base, path.posix.dirname(rel), serverOnly.marker));
+}
+
+const isWithinReal = (real, parent) => real === parent || real.startsWith(`${parent}${path.sep}`);
+
+// A link is followed only while it stays inside the base: one that leads out would dangle in a local container,
+// and following it would send what it points at. A link back to a folder the walk is already inside never ends.
+function statInside(full, realBase) {
+    const stat = fs.lstatSync(full);
+    if (!stat.isSymbolicLink()) return stat;
+    const real = fs.realpathSync(full);
+    if (!isWithinReal(real, realBase)) return null;
+    const target = fs.statSync(full);
+    return target.isDirectory() && isWithinReal(fs.realpathSync(path.dirname(full)), real) ? null : target;
+}
+
+// Every file below the roots that the rules keep, keyed by its path relative to base, and every folder:
+// the host creates folders before a step writes into them, empty ones included.
+export function walk(base, roots, rules, { outputs = false } = {}) {
+    const files = new Map();
+    const dirs = [];
+    let realBase;
+    try {
+        realBase = fs.realpathSync(base);
+    } catch {
+        return { files, dirs };
+    }
+    const visit = (rel, inRoot, serverOnlyRoot) => {
+        let stat;
+        try {
+            stat = statInside(path.join(base, rel), realBase);
+        } catch {
+            return;
+        }
+        if (!stat) return;
+        if (stat.isFile()) {
+            const kept = serverOnlyRoot === null
+                || rules.serverOnly.keep.some((pattern) => matchesPattern(rel.slice(serverOnlyRoot.length + 1), pattern));
+            if (kept) files.set(rel, { size: stat.size, mtimeMs: stat.mtimeMs, mode: stat.mode & 0o777 });
+            return;
+        }
+        if (!stat.isDirectory()) return;
+        let insideServerOnly = serverOnlyRoot;
+        if (insideServerOnly === null && isServerOnly(base, rel, rules)) {
+            if (!outputs) return;
+            insideServerOnly = rel;
+        }
+        if (insideServerOnly === null) dirs.push(rel);
+        for (const name of fs.readdirSync(path.join(base, rel)).sort()) {
+            const childInRoot = inRoot ? `${inRoot}/${name}` : name;
+            if (!isExcluded(childInRoot, rules)) visit(rel === '.' ? name : `${rel}/${name}`, childInRoot, insideServerOnly);
+        }
+    };
+    roots.forEach((root) => visit(root, '', null));
+    return { files, dirs };
+}
+
+export const snapshot = (base, roots, rules, options) => walk(base, roots, rules, options).files;
+
+// Hashes reused while size and mtime are unchanged; `index` is updated in place and pruned to `files`.
+export function hashIndex(base, files, index) {
+    const hashes = new Map();
+    for (const [rel, meta] of files) {
+        const cached = index.get(rel);
+        const sha256 = cached && cached.size === meta.size && cached.mtimeMs === meta.mtimeMs
+            ? cached.sha256
+            : sha256Of(fs.readFileSync(path.join(base, rel)));
+        index.set(rel, { size: meta.size, mtimeMs: meta.mtimeMs, sha256 });
+        hashes.set(rel, sha256);
+    }
+    [...index.keys()].filter((rel) => !files.has(rel)).forEach((rel) => index.delete(rel));
+    return hashes;
+}
+
+// A `*` segment stands for every folder at that level, so `b/*/p` names each conan package folder that exists.
+export function expandRoots(base, roots) {
+    const step = (partials, segment) => partials.flatMap((partial) => {
+        if (segment !== '*') return [partial ? `${partial}/${segment}` : segment];
+        const dir = path.join(base, partial);
+        if (!fs.existsSync(dir)) return [];
+        return fs.readdirSync(dir, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => (partial ? `${partial}/${entry.name}` : entry.name))
+            .sort();
+    });
+    return roots.flatMap((root) => root.split('/').reduce(step, ['']));
+}
+
+// The store units a machine already holds (a crate version, a conan package folder). They never change once
+// written, so the runner sends only the units missing there.
+export function presentUnits(base, outputRoots) {
+    return expandRoots(base, outputRoots.filter((root) => root.includes('*'))).filter((unit) => {
+        try {
+            return fs.readdirSync(path.join(base, unit)).length > 0;
+        } catch {
+            return false;
+        }
+    });
+}
+
+// Permission bits only: a synced file never becomes setuid, setgid or sticky.
+export const modeOf = (mode) => (Number.isInteger(mode) ? mode & 0o777 : 0o644);
+
+export function planSync(current, manifest) {
+    const write = Object.entries(manifest).filter(([rel, entry]) => current.get(rel) !== entry.sha256).map(([rel]) => rel);
+    const remove = [...current.keys()].filter((rel) => !(rel in manifest));
+    return { write, remove };
+}
+
+// Written files get the current time as mtime, so make and ninja rebuild what depends on them.
+export function applySync(base, plan, manifest, blobPathOf) {
+    plan.remove.forEach((rel) => fs.rmSync(path.join(base, rel), { force: true }));
+    plan.write.forEach((rel) => {
+        const dest = path.join(base, rel);
+        fs.mkdirSync(path.dirname(dest), { recursive: true });
+        fs.rmSync(dest, { force: true });
+        fs.copyFileSync(blobPathOf(manifest[rel].sha256), dest);
+        fs.chmodSync(dest, modeOf(manifest[rel].mode));
+    });
+}
+
+export function diffSnapshots(before, after) {
+    const changed = [...after].filter(([rel, meta]) => {
+        const old = before.get(rel);
+        return !old || old.size !== meta.size || old.mtimeMs !== meta.mtimeMs;
+    }).map(([rel]) => rel);
+    const removed = [...before.keys()].filter((rel) => !after.has(rel));
+    return { changed, removed };
+}
+
+// Characters a host path would read as structure: a backslash is a separator on Windows, where a colon
+// names a drive or a stream, and NUL ends a path anywhere.
+const UNSAFE_PATH = process.platform === 'win32' ? /[\\:\0]/ : /[\\\0]/;
+
+// The runner names files by container path; only a path inside one of the step's mounts maps back,
+// so the runner cannot make the client write anywhere else.
+export function hostPathOf(mounts, containerPath) {
+    const outside = () => new Error(`crossbind: the remote runner returned ${containerPath}, outside the mounts it was given.`);
+    const isClean = typeof containerPath === 'string' && !UNSAFE_PATH.test(containerPath)
+        && path.posix.normalize(containerPath) === containerPath;
+    const mount = isClean && mounts.find((m) => containerPath.startsWith(`${m.container}/`));
+    if (!mount) throw outside();
+    const rel = containerPath.slice(mount.container.length + 1);
+    const file = path.join(mount.host, rel);
+    const back = path.relative(mount.host, file);
+    if (back === '..' || back.startsWith(`..${path.sep}`) || path.isAbsolute(back)) throw outside();
+    return { mount, rel, file };
+}
+
+// Whether a step may hand this file back: it lies below one of the step's output roots, and below a store
+// root (`registry/src/*/*`) only inside a unit the machine does not hold yet, as those never change.
+export function isDeclaredOutput(rel, outputRoots, present = []) {
+    const parts = rel.split('/');
+    return outputRoots.some((root) => {
+        if (!root.includes('*')) return root === '.' || rel === root || rel.startsWith(`${root}/`);
+        const depth = root.split('/').length;
+        const unit = parts.slice(0, depth).join('/');
+        return parts.length > depth && matchesPattern(unit, root) && !present.includes(unit);
+    });
+}

package/src/runner/server.js

@@ -0,0 +1,320 @@
+import http from 'node:http';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import readline from 'node:readline';
+import { spawn } from 'node:child_process';
+import { pathToFileURL } from 'node:url';
+import { createBlobStore, isHash, isUploadId } from './blobs.js';
+import {
+    snapshot, hashIndex, planSync, applySync, diffSnapshots, expandRoots, sha256Of,
+} from './files.js';
+
+// The runner side of `RUNNER=REMOTE`: it runs inside a crossbind toolchain image and executes the
+// toolchain steps a client would otherwise `docker run`, against folders it keeps in sync with the client.
+
+// 2: a blob too large for one request arrives in parts.
+export const PROTOCOL_VERSION = 2;
+export const MIN_TOKEN_LENGTH = 16;
+const MAX_JSON_BYTES = 16 * 1024 * 1024;
+const INLINE_OUTPUT_BYTES = 8 * 1024 * 1024;
+const MAX_MOUNTS = 8;
+const DEFAULT_PORT = 8787;
+const CONTENT_RANGE = /^bytes (\d+)-(\d+)\/(\d+)$/;
+// A proxy in front of a hosted runner (fly, cloudflare) closes a response that stays quiet, and a compile
+// step can print nothing for minutes.
+const HEARTBEAT_MS = 15_000;
+// Where crossbind mounts things into its images: the project base, and the cargo and conan caches.
+const DEFAULT_MOUNT_PREFIXES = ['/tmp/crossbind/live', '/var/cache/crossbind'];
+// cargo runs from a directory outside every mount so its upward config search finds nothing.
+const DEFAULT_SCRATCH_DIRS = ['/tmp/crossbind-cargo'];
+
+const isStringArray = (value) => Array.isArray(value) && value.every((item) => typeof item === 'string');
+const isSafeRel = (rel) => typeof rel === 'string' && rel.length > 0 && !path.posix.isAbsolute(rel)
+    && path.posix.normalize(rel) === rel && !rel.split('/').includes('..');
+const isCleanAbsolute = (value) => typeof value === 'string' && path.posix.isAbsolute(value)
+    && path.posix.normalize(value) === value && (value === '/' || !value.endsWith('/'));
+const isWithin = (child, parent) => child === parent || child.startsWith(`${parent}/`);
+const isUnderRoot = (rel, root) => root === '.' || isWithin(rel, root);
+const isPlainName = (value) => typeof value === 'string' && /^[\w.-]+$/.test(value) && value !== '.' && value !== '..';
+const isNamePattern = (value) => isPlainName(typeof value === 'string' && value.endsWith('*') ? value.slice(0, -1) : value);
+// A build and a runner name one toolchain by digest even when they pull it from different registries (a mirror).
+const digestOf = (ref) => (typeof ref === 'string' && ref.includes('@') ? ref.slice(ref.indexOf('@') + 1) : ref);
+
+class RequestError extends Error {
+    constructor(status, message, extra = {}) {
+        super(message);
+        this.status = status;
+        this.extra = extra;
+    }
+}
+
+function send(res, status, body) {
+    res.writeHead(status, { 'content-type': 'application/json' });
+    res.end(JSON.stringify(body));
+}
+
+async function readJson(req) {
+    let size = 0;
+    const chunks = [];
+    for await (const chunk of req) {
+        size += chunk.length;
+        if (size > MAX_JSON_BYTES) throw new RequestError(413, 'request body too large');
+        chunks.push(chunk);
+    }
+    try {
+        return JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    } catch {
+        throw new RequestError(400, 'request body is not JSON');
+    }
+}
+
+function tokenMatches(header, token) {
+    const expected = Buffer.from(`Bearer ${token}`);
+    const actual = Buffer.from(header ?? '');
+    return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
+}
+
+function validateRules(rules) {
+    if (!rules || !isStringArray(rules.dirs) || !isStringArray(rules.extensions)) throw new RequestError(400, 'rules must list dirs and extensions');
+    if (rules.files !== undefined && !(isStringArray(rules.files) && rules.files.every(isNamePattern))) {
+        throw new RequestError(400, 'rules.files must list file names, each optionally ending in *');
+    }
+    const { serverOnly } = rules;
+    if (serverOnly === undefined) return;
+    if (!isStringArray(serverOnly.names) || !serverOnly.names.every(isPlainName) || !isPlainName(serverOnly.marker) || !isStringArray(serverOnly.keep)) {
+        throw new RequestError(400, 'rules.serverOnly must name folders, a marker file and keep patterns');
+    }
+}
+
+function validateMount(mount, mountPrefixes) {
+    if (!mount || !isCleanAbsolute(mount.path) || !mountPrefixes.some((prefix) => isWithin(mount.path, prefix))) {
+        throw new RequestError(400, `mount ${mount?.path} is not one this runner allows`);
+    }
+    const { roots, outputRoots, manifest, dirs } = mount;
+    if (!isStringArray(roots) || !roots.every(isSafeRel)) throw new RequestError(400, 'roots must be relative paths inside the mount');
+    if (!isStringArray(outputRoots) || !outputRoots.every(isSafeRel)) throw new RequestError(400, 'outputRoots must be relative paths inside the mount');
+    if (!manifest || typeof manifest !== 'object') throw new RequestError(400, 'manifest must be an object');
+    for (const [rel, entry] of Object.entries(manifest)) {
+        if (!isSafeRel(rel) || !roots.some((root) => isUnderRoot(rel, root))) throw new RequestError(400, `manifest path ${rel} is outside the roots`);
+        if (!isHash(entry?.sha256)) throw new RequestError(400, `manifest entry ${rel} has no sha256`);
+    }
+    if (!isStringArray(dirs) || !dirs.every((rel) => isSafeRel(rel) && roots.some((root) => isUnderRoot(rel, root)))) {
+        throw new RequestError(400, 'dirs must be folders inside the roots');
+    }
+    if (mount.present !== undefined && !(isStringArray(mount.present) && mount.present.every(isSafeRel))) {
+        throw new RequestError(400, 'present must list relative folders');
+    }
+}
+
+function validateExec(body, { mountPrefixes, scratchDirs, image, role }) {
+    if (body.role !== role) throw new RequestError(409, `this runner serves the ${role} toolchain, not ${body.role}`);
+    if (image && digestOf(body.image) !== digestOf(image)) throw new RequestError(409, `toolchain mismatch: runner has ${image}, build pins ${body.image}`);
+    validateRules(body.rules);
+    const { mounts, cwd, argv, env } = body;
+    if (!Array.isArray(mounts) || mounts.length === 0 || mounts.length > MAX_MOUNTS) throw new RequestError(400, `mounts must list 1 to ${MAX_MOUNTS} folders`);
+    mounts.forEach((mount) => validateMount(mount, mountPrefixes));
+    if (mounts.some((a, i) => mounts.some((b, j) => i !== j && isWithin(a.path, b.path)))) throw new RequestError(400, 'mounts must not overlap');
+    const allowedCwd = isCleanAbsolute(cwd) && [...mounts.map((mount) => mount.path), ...scratchDirs].some((dir) => isWithin(cwd, dir));
+    if (!allowedCwd) throw new RequestError(400, 'cwd must be inside a mount or a scratch folder');
+    if (!isStringArray(argv) || argv.length === 0) throw new RequestError(400, 'argv must be a non-empty string array');
+    if (!env || typeof env !== 'object' || !Object.values(env).every((value) => typeof value === 'string')) throw new RequestError(400, 'env must map names to strings');
+}
+
+// A command never sees the runner's own settings, its token above all.
+const commandEnv = (env) => ({
+    ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('CROSSBIND_RUNNER_'))),
+    ...env,
+});
+
+// The job count a client wrote for its own machine means nothing here: a step runs as many jobs as this
+// runner has cores.
+const withJobs = (argv, jobs) => argv.map((arg, i) => {
+    if (/^-j\d+$/.test(arg)) return `-j${jobs}`;
+    return /^\d+$/.test(arg) && argv[i - 1] === '-j' ? String(jobs) : arg;
+});
+
+function runCommand({ argv: given, cwd, env }, line, jobs) {
+    const argv = withJobs(given, jobs);
+    return new Promise((resolve) => {
+        const child = spawn(argv[0], argv.slice(1), { cwd, env: commandEnv(env) });
+        child.stdout.on('data', (data) => line({ stdout: data.toString() }));
+        child.stderr.on('data', (data) => line({ stderr: data.toString() }));
+        child.on('error', (error) => {
+            line({ stderr: `crossbind runner: ${error.message}\n` });
+            resolve(127);
+        });
+        child.on('close', (code) => resolve(code ?? 1));
+    });
+}
+
+export function createRunnerServer({
+    mountPrefixes = DEFAULT_MOUNT_PREFIXES, scratchDirs = DEFAULT_SCRATCH_DIRS, blobDir, token, image, role, heartbeatMs = HEARTBEAT_MS,
+    jobs = os.availableParallelism(),
+}) {
+    if (!token) throw new Error('crossbind runner: a token is required - the runner runs build commands for whoever holds it.');
+    if (token.length < MIN_TOKEN_LENGTH) throw new Error(`crossbind runner: the token needs at least ${MIN_TOKEN_LENGTH} characters.`);
+    const blobs = createBlobStore(blobDir);
+    const indexes = new Map();
+    const indexFor = (mountPath) => {
+        if (!indexes.has(mountPath)) indexes.set(mountPath, new Map());
+        return indexes.get(mountPath);
+    };
+    let queue = Promise.resolve();
+
+    function syncMount(mount, rules) {
+        fs.mkdirSync(mount.path, { recursive: true });
+        const current = hashIndex(mount.path, snapshot(mount.path, mount.roots, rules), indexFor(mount.path));
+        applySync(mount.path, planSync(current, mount.manifest), mount.manifest, blobs.pathOf);
+        // The host creates folders before a step writes into them (`-o build/bridge/x.cpp`), so even empty ones travel.
+        mount.dirs.forEach((rel) => fs.mkdirSync(path.join(mount.path, rel), { recursive: true }));
+    }
+
+    // One command at a time: each mount is a single tree, as with a local docker run.
+    async function execute(body, line) {
+        body.mounts.forEach((mount) => syncMount(mount, body.rules));
+        // Outputs come back whatever their type: the glue JavaScript a link writes is the build's product.
+        const outputRules = { ...body.rules, extensions: [] };
+        const isStore = (root) => root.includes('*');
+        const outputsOf = (mount) => snapshot(mount.path, mount.outputRoots.filter((root) => !isStore(root)), outputRules, { outputs: true });
+        // A store unit (a crate version, a conan package folder) never changes once written, so every unit the
+        // client does not report travels whole, touched by this command or not, and the rest are not walked.
+        const storeFilesOf = (mount) => {
+            const present = new Set(mount.present ?? []);
+            const missing = expandRoots(mount.path, mount.outputRoots.filter(isStore)).filter((unit) => !present.has(unit));
+            return snapshot(mount.path, missing, outputRules, { outputs: true });
+        };
+        const before = body.mounts.map(outputsOf);
+        fs.mkdirSync(body.cwd, { recursive: true });
+        const exit = await runCommand(body, line, jobs);
+
+        const outputs = {};
+        const removed = [];
+        body.mounts.forEach((mount, i) => {
+            const diffed = outputsOf(mount);
+            const store = storeFilesOf(mount);
+            const after = new Map([...diffed, ...store]);
+            const diff = diffSnapshots(before[i], diffed);
+            new Set([...diff.changed, ...store.keys()]).forEach((rel) => {
+                const file = `${mount.path}/${rel}`;
+                const data = fs.readFileSync(file);
+                const entry = { sha256: blobs.putBuffer(data), mode: after.get(rel).mode, size: data.length };
+                outputs[file] = entry;
+                // Small outputs ride along in the stream; a larger one is fetched by its hash afterwards.
+                if (data.length <= INLINE_OUTPUT_BYTES) line({ file, ...entry, data: data.toString('base64') });
+            });
+            removed.push(...diff.removed.map((rel) => `${mount.path}/${rel}`));
+        });
+        return { exit, outputs, removed };
+    }
+
+    async function handleExec(req, res) {
+        const body = await readJson(req);
+        validateExec(body, {
+            mountPrefixes, scratchDirs, image, role,
+        });
+        const hashes = body.mounts.flatMap((mount) => Object.values(mount.manifest).map((entry) => entry.sha256));
+        const missing = [...new Set(hashes)].filter((hash) => !blobs.has(hash));
+        if (missing.length > 0) throw new RequestError(409, 'missing blobs', { missing });
+        // The answer starts at once and heartbeats until the end: a step can wait behind another one longer than
+        // a proxy, or the client's fetch, waits for a response.
+        res.writeHead(200, { 'content-type': 'application/x-ndjson' });
+        res.flushHeaders();
+        const line = (record) => res.write(`${JSON.stringify(record)}\n`);
+        const heartbeat = setInterval(() => line({ heartbeat: true }), heartbeatMs);
+        try {
+            const run = queue.then(() => execute(body, line));
+            queue = run.catch(() => {});
+            res.end(`${JSON.stringify(await run)}\n`);
+        } finally {
+            clearInterval(heartbeat);
+        }
+    }
+
+    // One JSON line per blob ({sha256, data: base64}), so a cold start uploads its inputs in a few requests.
+    async function handleBatchUpload(req, res) {
+        let stored = 0;
+        for await (const text of readline.createInterface({ input: req, crlfDelay: Infinity })) {
+            if (!text) continue;
+            const { sha256, data } = JSON.parse(text);
+            const bytes = Buffer.from(data ?? '', 'base64');
+            if (!isHash(sha256) || sha256Of(bytes) !== sha256) throw new RequestError(400, `content does not match ${sha256}`);
+            blobs.putBuffer(bytes);
+            stored += 1;
+        }
+        send(res, 201, { stored });
+    }
+
+    async function route(req, res) {
+        const url = new URL(req.url, 'http://runner');
+        if (req.method === 'GET' && url.pathname === '/v1/health') return send(res, 200, { ok: true, protocol: PROTOCOL_VERSION, role, image });
+        if (!tokenMatches(req.headers.authorization, token)) return send(res, 401, { error: 'invalid token' });
+
+        if (req.method === 'POST' && url.pathname === '/v1/blobs') return handleBatchUpload(req, res);
+        const blobMatch = url.pathname.match(/^\/v1\/blobs\/([^/]+)$/);
+        if (blobMatch) {
+            const hash = decodeURIComponent(blobMatch[1]);
+            if (!isHash(hash)) return send(res, 400, { error: 'blob names are sha256 digests' });
+            if (req.method === 'PUT' && req.headers['content-range'] !== undefined) {
+                const range = CONTENT_RANGE.exec(req.headers['content-range']);
+                const [start, end, total] = range ? range.slice(1).map(Number) : [];
+                const upload = req.headers['x-crossbind-upload'];
+                if (!range || start > end || end >= total || !isUploadId(upload)) {
+                    return send(res, 400, { error: 'a part needs a content-range of bytes start-end/total and an x-crossbind-upload id' });
+                }
+                const result = await blobs.putPart(hash, upload, { start, end, total }, req);
+                if (result.status === 'stored') return send(res, 201, { stored: hash });
+                if (result.status === 'partial') return send(res, 202, { received: result.received });
+                if (result.status === 'gap') return send(res, 409, { error: `the upload holds ${result.received} bytes, so its next part starts there`, received: result.received });
+                return send(res, 400, { error: result.status === 'short' ? 'the part is not as long as its content-range says' : 'content does not match the hash' });
+            }
+            if (req.method === 'PUT') {
+                return (await blobs.putStream(hash, req)) ? send(res, 201, { stored: hash }) : send(res, 400, { error: 'content does not match the hash' });
+            }
+            if (req.method === 'GET') {
+                if (!blobs.has(hash)) return send(res, 404, { error: 'unknown blob' });
+                res.writeHead(200, { 'content-type': 'application/octet-stream' });
+                return fs.createReadStream(blobs.pathOf(hash)).pipe(res);
+            }
+        }
+        if (req.method === 'POST' && url.pathname === '/v1/missing') {
+            const { hashes } = await readJson(req);
+            if (!isStringArray(hashes) || !hashes.every(isHash)) return send(res, 400, { error: 'hashes must be sha256 digests' });
+            return send(res, 200, { missing: hashes.filter((hash) => !blobs.has(hash)) });
+        }
+        if (req.method === 'POST' && url.pathname === '/v1/exec') return handleExec(req, res);
+        return send(res, 404, { error: 'not found' });
+    }
+
+    return http.createServer((req, res) => {
+        route(req, res).catch((error) => {
+            if (res.headersSent) {
+                res.end(`${JSON.stringify({ exit: 1, error: error.message })}\n`);
+                return;
+            }
+            send(res, error.status ?? 500, { error: error.message, ...error.extra });
+        });
+    });
+}
+
+const listFromEnv = (value, fallback) => (value ? value.split(',').map((item) => item.trim()).filter(Boolean) : fallback);
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+    const token = process.env.CROSSBIND_RUNNER_TOKEN;
+    delete process.env.CROSSBIND_RUNNER_TOKEN;
+    if (!token || token.length < MIN_TOKEN_LENGTH) {
+        process.stderr.write(`crossbind runner: set CROSSBIND_RUNNER_TOKEN to at least ${MIN_TOKEN_LENGTH} characters - the runner runs build commands for whoever holds it.\n`);
+        process.exit(1);
+    }
+    const server = createRunnerServer({
+        mountPrefixes: listFromEnv(process.env.CROSSBIND_RUNNER_MOUNT_PREFIXES, DEFAULT_MOUNT_PREFIXES),
+        scratchDirs: listFromEnv(process.env.CROSSBIND_RUNNER_SCRATCH_DIRS, DEFAULT_SCRATCH_DIRS),
+        blobDir: process.env.CROSSBIND_RUNNER_BLOBS || '/tmp/crossbind-runner/blobs',
+        token,
+        image: process.env.CROSSBIND_RUNNER_IMAGE || '',
+        role: process.env.CROSSBIND_RUNNER_ROLE || 'web',
+    });
+    const port = Number(process.env.CROSSBIND_RUNNER_PORT || DEFAULT_PORT);
+    server.listen(port, '0.0.0.0', () => process.stdout.write(`crossbind runner (${process.env.CROSSBIND_RUNNER_ROLE || 'web'}) listening on ${port}\n`));
+}

package/src/state/attachConanDependencies.js

@@ -0,0 +1,56 @@
+import { getFilledConfig, flattenDependencies } from './loadConfig.js';
+import calculateDependencyParameters from './calculateDependencyParameters.js';
+import { conanStageDir, conanPackageDir } from '../utils/conanImport.js';
+import { conanDependenciesKey } from '../utils/conanDependencies.js';
+import { readConanManifest, conanTargetName } from '../utils/conanStage.js';
+import { TARGETS, targetPathOf } from '../utils/targets.js';
+
+// What a recipe declares for its consumers, per staged target since it declares other things on another
+// platform: the defines your code compiles with, and the system libraries and frameworks a Node.js addon
+// or a native executable links. A framework goes as one linker argument: CMake collapses a repeated
+// -framework. A debug build takes the release packages, so the build type is left out, and a target's
+// specs stay one object, as getData takes them.
+function targetSpecsOf(name, targets) {
+    return Object.entries(targets).flatMap(([targetPath, packages]) => {
+        const target = TARGETS.find((t) => targetPathOf(t) === targetPath);
+        const pkg = packages.find((entry) => entry.name === name);
+        if (!target || !pkg) return [];
+        const compileOptions = pkg.defines.map((define) => `-D${define}`);
+        const addonFlags = [...pkg.frameworks.map((framework) => `-Wl,-framework,${framework}`), ...pkg.systemLibs.map((lib) => `-l${lib}`)];
+        const specs = {
+            ...(compileOptions.length > 0 ? { cmake: { compileOptions } } : {}),
+            ...(addonFlags.length > 0 ? { binary: { addonFlags } } : {}),
+        };
+        if (Object.keys(specs).length === 0) return [];
+        return [{
+            platform: target.platform, arch: target.arch, runtime: target.runtime, specs,
+        }];
+    });
+}
+
+// Every library a recipe names on some target (libpng is png16 on Windows): a target links the archives
+// staged for it, so a name it lacks drops out there.
+const libsOf = (name, targets) => [...new Set(Object.values(targets).flat().filter((entry) => entry.name === name).flatMap((entry) => entry.libs))];
+
+export function conanDependencyConfigs(manifest, stageDir) {
+    return manifest.packages.map((pkg) => ({
+        general: { name: conanTargetName(pkg.name), conan: pkg },
+        export: { type: 'cmake', libName: libsOf(pkg.name, manifest.targets) },
+        paths: { project: conanPackageDir(stageDir, pkg.name), output: `${conanPackageDir(stageDir, pkg.name)}/dist` },
+        targetSpecs: targetSpecsOf(pkg.name, manifest.targets),
+    }));
+}
+
+// The staged Conan packages join the config as dependencies, so header resolution, the link, the
+// cmake graph and the license rows see them like any port. Only packages staged for the current
+// conanDependencies join; reading the manifest runs nothing, so state can do it while it loads.
+export default function attachConanDependencies(config) {
+    const isDeclared = Object.keys(config.conanDependencies ?? {}).length > 0;
+    if (!isDeclared && !config.dependencies?.some((d) => d.general.conan)) return;
+    const stageDir = conanStageDir(config.paths.cache);
+    const manifest = isDeclared ? readConanManifest(stageDir, conanDependenciesKey(config.conanDependencies)) : null;
+    const conan = manifest ? conanDependencyConfigs(manifest, stageDir).map((d) => getFilledConfig(d, { isDepend: true })) : [];
+    config.dependencies = [...config.dependencies.filter((d) => !d.general.conan), ...conan];
+    config.allDependencies = flattenDependencies(config.dependencies);
+    config.dependencyParameters = calculateDependencyParameters(config);
+}

package/src/state/index.js

@@ -1,8 +1,13 @@
+import fs from 'node:fs';
 import loadJson from '../utils/loadJson.js';
 import writeJson from '../utils/writeJson.js';
-import { TARGETS, targetPathOf, nodeAddonNamesOf } from '../utils/targets.js';
+import {
+    TARGETS, targetPathOf, nodeAddonNamesOf, nativeCommandNamesOf, wasiNamesOf,
+} from '../utils/targets.js';
 import { findCargoModuleImportsIn, writeCargoMarker } from '../utils/cargoImport.js';
-import loadConfig from './loadConfig.js';
+import { exportWasiSdkPath } from '../utils/wasiToolchain.js';
+import loadConfig, { assertBinarySelection } from './loadConfig.js';
+import refreshConanDependencies from './refreshConanDependencies.js';
 
 const cacheDir = `${process.cwd()}/.crossbind`;
 
@@ -21,12 +26,9 @@ await initProcessState();
 async function initProcessState() {
     state.cache = loadCacheState();
     state.config = await loadConfig();
-    // Recipes read the wasi-sdk location from the environment (lazily, in
-    // getBuildParams); mirror the system-config value there so both sources
-    // behave the same.
-    if (state.config.system.WASI_SDK_PATH && !process.env.CROSSBIND_WASI_SDK_PATH) {
-        process.env.CROSSBIND_WASI_SDK_PATH = state.config.system.WASI_SDK_PATH;
-    }
+    assertBinarySelection(state.config);
+    refreshConanDependencies(state.config);
+    exportWasiSdkPath(state.config.system);
     // Same idea for the docker image overrides: pullDockerImage.js resolves them from the
     // environment alone, so it stays free of a state import (and of the cycle that would create).
     Object.entries({
@@ -52,12 +54,14 @@ async function initProcessState() {
             target.dataTxtName = `${state.config.general.name}-${target.path}.${target.runtimeEnv}.data.txt`;
         }
         if (target.platform === 'wasi') {
-            // A wasi build is a single command module - no JS glue, no preload.
-            target.wasmName = `${state.config.general.name}-${target.path}.wasm`;
+            Object.assign(target, wasiNamesOf(target, state.config.general.name));
         }
         if (target.runtimeEnv === 'node' && target.platform !== 'wasm') {
             Object.assign(target, nodeAddonNamesOf(target, state.config.general.name));
         }
+        if (target.runtimeEnv === 'native') {
+            Object.assign(target, nativeCommandNamesOf(target, state.config.general.name));
+        }
     });
 
     setAllDependecyPaths();
@@ -89,7 +93,12 @@ export function setAllDependecyPaths() {
         state.config.allDependencyPaths[target.path] = { cmake: {} };
         state.config.allDependencies.forEach((d) => {
             state.config.allDependencyPaths[target.path].cmake[d.general.name] = `${d.paths.output}/prebuilt`;
-            d.export.libName.forEach((name) => {
+            // A Conan recipe can name a library on another platform only (libpng is png16 on Windows).
+            const libNames = d.general.conan
+                ? d.export.libName.filter((name) => [target.path, target.releasePath]
+                    .some((targetPath) => fs.existsSync(`${d.paths.output}/prebuilt/${targetPath}/lib/lib${name}.a`)))
+                : d.export.libName;
+            libNames.forEach((name) => {
                 state.config.allDependencyPaths[target.path][name] = {
                     root: `${d.paths.output}/prebuilt/${target.path}`,
                 };