crossbind 2.0.0-beta.60 → 2.0.0-beta.63

package/src/integration/getDependFilePath.js

@@ -1,6 +1,9 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import state from '../state/index.js';
+import { parseCargoImport, writeCargoMarker } from '../utils/cargoImport.js';
+import { CONAN_PREFIX, resolveConanHeader } from '../utils/conanImport.js';
+import refreshConanDependencies from '../state/refreshConanDependencies.js';
 
 const LAYOUT = { header: 'include', module: 'swig', source: '' };
 
@@ -76,25 +79,28 @@ function getFolderCandidates(pkg, alias) {
 }
 
 export default function getDependFilePath(source, target) {
-    // `cargo:<crate>` imports (the node:/npm: convention for a non-npm store) resolve to a
-    // generated marker .rs; the transformer turns it into the crate-import bridge (no surface
-    // file, no package). The scheme is unambiguous, so npm names never collide and an
-    // undeclared crate is a hard, actionable error instead of a silent npm fallthrough.
+    // `cargo:<crate>[/<module>...]` imports (the node:/npm: convention for a non-npm store)
+    // resolve to a generated marker .rs; the transformer turns it into the crate-import bridge
+    // (no surface file, no package). The scheme is unambiguous, so npm names never collide and
+    // an undeclared crate is a hard, actionable error instead of a silent npm fallthrough.
     if (source.startsWith('cargo:')) {
-        const crateName = source.slice('cargo:'.length);
+        const cargoImport = parseCargoImport(source);
         const cargoDeps = state.config.cargoDependencies ?? {};
-        if (!Object.hasOwn(cargoDeps, crateName)) {
-            throw new Error(`crossbind: '${source}' is not declared - add '${crateName}' to cargoDependencies in crossbind.config.`);
-        }
-        const marker = path.join(state.config.paths.cache, 'rust-crates', `${crateName}.rs`);
-        if (!fs.existsSync(marker)) {
-            fs.mkdirSync(path.dirname(marker), { recursive: true });
-            fs.writeFileSync(marker, `// crossbind cargo crate import marker: ${crateName}\n`);
+        if (!Object.hasOwn(cargoDeps, cargoImport.crateName)) {
+            throw new Error(`crossbind: '${source}' is not declared - add '${cargoImport.crateName}' to cargoDependencies in crossbind.config.`);
         }
+        const marker = writeCargoMarker(state.config.paths.cache, cargoImport);
         existsCache.set(marker, true);
         return marker;
     }
 
+    // `conan:<package>/<header>` names a header of a declared Conan package, staged per target like a
+    // port's prebuilt; from there the header takes the same path as any other.
+    if (source.startsWith(CONAN_PREFIX)) {
+        refreshConanDependencies(state.config);
+        return resolveConanHeader(state.config, source, target ?? state.targets.find((t) => t.platform === 'wasm'));
+    }
+
     const matches = getMatchingPackages(source);
     if (matches.length === 0) return null;
 

package/src/integration/resolveNativeImport.js

@@ -0,0 +1,14 @@
+import { createRequire } from 'node:module';
+import upath from 'upath';
+import getDependFilePath from './getDependFilePath.js';
+
+// A package that is not a crossbind dependency (the conformance kit) resolves the way the bundler finds it.
+export default function resolveNativeImport(specifier, importer, target) {
+    if (specifier.startsWith('.')) return upath.resolve(upath.dirname(importer), specifier);
+    if (upath.isAbsolute(specifier)) return specifier;
+    try {
+        return getDependFilePath(specifier, target) ?? createRequire(importer).resolve(specifier);
+    } catch (e) {
+        return null;
+    }
+}

package/src/node/dev.mjs

@@ -0,0 +1,19 @@
+// `node --import crossbind/node/dev app.mjs`: builds this machine's binary when the app's native sources or imports
+// changed since the last build it ran, then serves the app's native imports through the hooks that build wrote.
+import { pathToFileURL } from 'node:url';
+import loadConfig from '../state/loadConfig.js';
+import { DEV_BUILD_ENV, ensureDevBuild, projectDirOf } from '../utils/nodeDevBuild.js';
+
+if (!process.env[DEV_BUILD_ENV]) {
+    // node --watch restarts when a build rewrites the outputs the last start loaded. Its SIGTERM waits for the build
+    // and its record, then ends this start: the next one finds the build done instead of starting it over.
+    let restarting = false;
+    const restart = () => { restarting = true; };
+    process.on('SIGTERM', restart);
+    const hooks = ensureDevBuild(await loadConfig(projectDirOf(process.argv[1])));
+    // A signal that came during the build is handled on the next turn of the event loop.
+    await new Promise((resolve) => { setImmediate(resolve); });
+    process.off('SIGTERM', restart);
+    if (restarting) process.exit(143);
+    if (hooks) await import(pathToFileURL(hooks).href);
+}

package/src/runner/blobs.js

@@ -0,0 +1,135 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { pipeline } from 'node:stream/promises';
+import { sha256Of } from './files.js';
+
+const SHA256 = /^[0-9a-f]{64}$/;
+const UPLOAD_ID = /^[\w-]{8,64}$/;
+// A part nobody added to for an hour belongs to an upload that stopped; the disk may be memory (Cloud Run).
+const STALE_PART_MS = 60 * 60 * 1000;
+
+export const isHash = (value) => typeof value === 'string' && SHA256.test(value);
+export const isUploadId = (value) => typeof value === 'string' && UPLOAD_ID.test(value);
+
+// Content-addressed: a blob is stored under its sha256, and an upload whose bytes hash to something else is dropped.
+export function createBlobStore(dir) {
+    fs.mkdirSync(dir, { recursive: true });
+    const root = `${path.resolve(dir)}${path.sep}`;
+
+    const pathOf = (hash) => {
+        if (!isHash(hash)) throw new Error(`crossbind runner: "${hash}" is not a sha256 digest`);
+        return path.join(dir, hash);
+    };
+
+    const has = (hash) => isHash(hash) && fs.existsSync(path.join(dir, hash));
+
+    async function putStream(hash, stream) {
+        const temp = path.join(dir, `.${hash}.${process.pid}.${crypto.randomUUID()}`);
+        const digest = crypto.createHash('sha256');
+        const output = fs.createWriteStream(temp);
+        try {
+            await pipeline(stream, async function* hashing(source) {
+                for await (const chunk of source) {
+                    digest.update(chunk);
+                    yield chunk;
+                }
+            }, output);
+        } catch (error) {
+            // pipeline settles before a file that is still opening closes; removed earlier, it would come back.
+            if (!output.closed) await new Promise((resolve) => { output.once('close', resolve); });
+            fs.rmSync(temp, { force: true });
+            throw error;
+        }
+        if (digest.digest('hex') !== hash) {
+            fs.rmSync(temp, { force: true });
+            return false;
+        }
+        fs.renameSync(temp, pathOf(hash));
+        return true;
+    }
+
+    function dropStaleParts() {
+        const now = Date.now();
+        for (const name of fs.readdirSync(dir).filter((file) => file.endsWith('.part'))) {
+            const stat = fs.statSync(path.join(dir, name), { throwIfNoEntry: false });
+            if (stat && now - stat.mtimeMs > STALE_PART_MS) fs.rmSync(path.join(dir, name), { force: true });
+        }
+    }
+
+    // A request names the part file, so its path is checked to stay in the store.
+    const partOf = (hash, upload) => {
+        const file = path.resolve(root, `.${hash}.${upload}.part`);
+        if (!file.startsWith(root)) throw new Error(`crossbind runner: "${hash}" and "${upload}" name no part in the store`);
+        return file;
+    };
+
+    // A blob too large for one request arrives in parts, in order and under the id of its upload; the last part
+    // checks the joined file against the hash. A part that breaks off leaves the ones before it, so the client sends
+    // that part again, and a part of a blob already joined answers as stored, since the last answer may have been lost.
+    async function writePart(hash, upload, { start, end, total }, stream) {
+        if (!isUploadId(upload)) throw new Error(`crossbind runner: "${upload}" is not an upload id`);
+        if (has(hash)) {
+            for await (const chunk of stream) void chunk;
+            return { status: 'stored' };
+        }
+        if (start === 0) dropStaleParts();
+        const partial = partOf(hash, upload);
+        const received = fs.existsSync(partial) ? fs.statSync(partial).size : 0;
+        if (start !== received) return { status: 'gap', received };
+        const restore = () => {
+            if (start === 0) fs.rmSync(partial, { force: true });
+            else if (fs.existsSync(partial)) fs.truncateSync(partial, start);
+        };
+        let length = 0;
+        const output = fs.createWriteStream(partial, { flags: 'a' });
+        try {
+            await pipeline(stream, async function* counting(source) {
+                for await (const chunk of source) {
+                    length += chunk.length;
+                    yield chunk;
+                }
+            }, output);
+        } catch (error) {
+            if (!output.closed) await new Promise((resolve) => { output.once('close', resolve); });
+            restore();
+            throw error;
+        }
+        if (length !== end - start + 1) {
+            restore();
+            return { status: 'short' };
+        }
+        if (end + 1 < total) return { status: 'partial', received: end + 1 };
+        const digest = crypto.createHash('sha256');
+        for await (const chunk of fs.createReadStream(partial)) digest.update(chunk);
+        const joined = digest.digest('hex');
+        if (joined !== hash) {
+            fs.rmSync(partial, { force: true });
+            return { status: 'mismatch' };
+        }
+        fs.renameSync(partial, pathOf(joined));
+        return { status: 'stored' };
+    }
+
+    // The parts of one upload go one at a time: a part whose connection broke off can still be writing when the client
+    // sends it again.
+    const writing = new Map();
+    function putPart(hash, upload, range, stream) {
+        const key = `${hash}.${upload}`;
+        const current = (writing.get(key) ?? Promise.resolve()).catch(() => {}).then(() => writePart(hash, upload, range, stream));
+        writing.set(key, current);
+        const forget = () => {
+            if (writing.get(key) === current) writing.delete(key);
+        };
+        current.then(forget, forget);
+        return current;
+    }
+
+    function putBuffer(data) {
+        const hash = sha256Of(data);
+        if (!has(hash)) fs.writeFileSync(pathOf(hash), data);
+        return hash;
+    }
+
+    return { pathOf, has, putStream, putPart, putBuffer };
+}

package/src/runner/files.js

@@ -0,0 +1,193 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+// Shared by the client (utils/remoteClient.js) and the runner (server.js). The runner reaches the toolchain
+// image as this folder alone, mounted or copied in, so nothing here may import from outside it.
+
+export const sha256Of = (data) => crypto.createHash('sha256').update(data).digest('hex');
+
+// `name` against a file name pattern: a trailing `*` matches any rest (`.env.*`).
+const matchesName = (name, pattern) => (pattern.endsWith('*') ? name.startsWith(pattern.slice(0, -1)) : name === pattern);
+
+// `rel` is relative to the root being walked, so a root that itself sits in node_modules (an installed port) still counts.
+export function isExcluded(rel, rules) {
+    const segments = rel.split('/');
+    if (segments.some((segment) => rules.dirs.includes(segment))) return true;
+    if (rules.files?.some((pattern) => matchesName(segments.at(-1), pattern))) return true;
+    return rules.extensions.includes(path.posix.extname(rel));
+}
+
+// Segment by segment: `*` matches any one segment, `*.a` any segment with that ending.
+export function matchesPattern(rel, pattern) {
+    const parts = rel.split('/');
+    const wanted = pattern.split('/');
+    return parts.length === wanted.length && wanted.every((want, i) => want === '*'
+        || (want.startsWith('*.') ? parts[i].endsWith(want.slice(1)) : want === parts[i]));
+}
+
+// A build tree such as a cargo target folder stays on the runner: a sync walk skips it, so it is neither
+// uploaded nor deleted, and an output walk returns only the artifacts its `keep` patterns name.
+function isServerOnly(base, rel, rules) {
+    const serverOnly = rules.serverOnly;
+    if (!serverOnly?.names.includes(path.posix.basename(rel))) return false;
+    return fs.existsSync(path.join(base, path.posix.dirname(rel), serverOnly.marker));
+}
+
+const isWithinReal = (real, parent) => real === parent || real.startsWith(`${parent}${path.sep}`);
+
+// A link is followed only while it stays inside the base: one that leads out would dangle in a local container,
+// and following it would send what it points at. A link back to a folder the walk is already inside never ends.
+function statInside(full, realBase) {
+    const stat = fs.lstatSync(full);
+    if (!stat.isSymbolicLink()) return stat;
+    const real = fs.realpathSync(full);
+    if (!isWithinReal(real, realBase)) return null;
+    const target = fs.statSync(full);
+    return target.isDirectory() && isWithinReal(fs.realpathSync(path.dirname(full)), real) ? null : target;
+}
+
+// Every file below the roots that the rules keep, keyed by its path relative to base, and every folder:
+// the host creates folders before a step writes into them, empty ones included.
+export function walk(base, roots, rules, { outputs = false } = {}) {
+    const files = new Map();
+    const dirs = [];
+    let realBase;
+    try {
+        realBase = fs.realpathSync(base);
+    } catch {
+        return { files, dirs };
+    }
+    const visit = (rel, inRoot, serverOnlyRoot) => {
+        let stat;
+        try {
+            stat = statInside(path.join(base, rel), realBase);
+        } catch {
+            return;
+        }
+        if (!stat) return;
+        if (stat.isFile()) {
+            const kept = serverOnlyRoot === null
+                || rules.serverOnly.keep.some((pattern) => matchesPattern(rel.slice(serverOnlyRoot.length + 1), pattern));
+            if (kept) files.set(rel, { size: stat.size, mtimeMs: stat.mtimeMs, mode: stat.mode & 0o777 });
+            return;
+        }
+        if (!stat.isDirectory()) return;
+        let insideServerOnly = serverOnlyRoot;
+        if (insideServerOnly === null && isServerOnly(base, rel, rules)) {
+            if (!outputs) return;
+            insideServerOnly = rel;
+        }
+        if (insideServerOnly === null) dirs.push(rel);
+        for (const name of fs.readdirSync(path.join(base, rel)).sort()) {
+            const childInRoot = inRoot ? `${inRoot}/${name}` : name;
+            if (!isExcluded(childInRoot, rules)) visit(rel === '.' ? name : `${rel}/${name}`, childInRoot, insideServerOnly);
+        }
+    };
+    roots.forEach((root) => visit(root, '', null));
+    return { files, dirs };
+}
+
+export const snapshot = (base, roots, rules, options) => walk(base, roots, rules, options).files;
+
+// Hashes reused while size and mtime are unchanged; `index` is updated in place and pruned to `files`.
+export function hashIndex(base, files, index) {
+    const hashes = new Map();
+    for (const [rel, meta] of files) {
+        const cached = index.get(rel);
+        const sha256 = cached && cached.size === meta.size && cached.mtimeMs === meta.mtimeMs
+            ? cached.sha256
+            : sha256Of(fs.readFileSync(path.join(base, rel)));
+        index.set(rel, { size: meta.size, mtimeMs: meta.mtimeMs, sha256 });
+        hashes.set(rel, sha256);
+    }
+    [...index.keys()].filter((rel) => !files.has(rel)).forEach((rel) => index.delete(rel));
+    return hashes;
+}
+
+// A `*` segment stands for every folder at that level, so `b/*/p` names each conan package folder that exists.
+export function expandRoots(base, roots) {
+    const step = (partials, segment) => partials.flatMap((partial) => {
+        if (segment !== '*') return [partial ? `${partial}/${segment}` : segment];
+        const dir = path.join(base, partial);
+        if (!fs.existsSync(dir)) return [];
+        return fs.readdirSync(dir, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => (partial ? `${partial}/${entry.name}` : entry.name))
+            .sort();
+    });
+    return roots.flatMap((root) => root.split('/').reduce(step, ['']));
+}
+
+// The store units a machine already holds (a crate version, a conan package folder). They never change once
+// written, so the runner sends only the units missing there.
+export function presentUnits(base, outputRoots) {
+    return expandRoots(base, outputRoots.filter((root) => root.includes('*'))).filter((unit) => {
+        try {
+            return fs.readdirSync(path.join(base, unit)).length > 0;
+        } catch {
+            return false;
+        }
+    });
+}
+
+// Permission bits only: a synced file never becomes setuid, setgid or sticky.
+export const modeOf = (mode) => (Number.isInteger(mode) ? mode & 0o777 : 0o644);
+
+export function planSync(current, manifest) {
+    const write = Object.entries(manifest).filter(([rel, entry]) => current.get(rel) !== entry.sha256).map(([rel]) => rel);
+    const remove = [...current.keys()].filter((rel) => !(rel in manifest));
+    return { write, remove };
+}
+
+// Written files get the current time as mtime, so make and ninja rebuild what depends on them.
+export function applySync(base, plan, manifest, blobPathOf) {
+    plan.remove.forEach((rel) => fs.rmSync(path.join(base, rel), { force: true }));
+    plan.write.forEach((rel) => {
+        const dest = path.join(base, rel);
+        fs.mkdirSync(path.dirname(dest), { recursive: true });
+        fs.rmSync(dest, { force: true });
+        fs.copyFileSync(blobPathOf(manifest[rel].sha256), dest);
+        fs.chmodSync(dest, modeOf(manifest[rel].mode));
+    });
+}
+
+export function diffSnapshots(before, after) {
+    const changed = [...after].filter(([rel, meta]) => {
+        const old = before.get(rel);
+        return !old || old.size !== meta.size || old.mtimeMs !== meta.mtimeMs;
+    }).map(([rel]) => rel);
+    const removed = [...before.keys()].filter((rel) => !after.has(rel));
+    return { changed, removed };
+}
+
+// Characters a host path would read as structure: a backslash is a separator on Windows, where a colon
+// names a drive or a stream, and NUL ends a path anywhere.
+const UNSAFE_PATH = process.platform === 'win32' ? /[\\:\0]/ : /[\\\0]/;
+
+// The runner names files by container path; only a path inside one of the step's mounts maps back,
+// so the runner cannot make the client write anywhere else.
+export function hostPathOf(mounts, containerPath) {
+    const outside = () => new Error(`crossbind: the remote runner returned ${containerPath}, outside the mounts it was given.`);
+    const isClean = typeof containerPath === 'string' && !UNSAFE_PATH.test(containerPath)
+        && path.posix.normalize(containerPath) === containerPath;
+    const mount = isClean && mounts.find((m) => containerPath.startsWith(`${m.container}/`));
+    if (!mount) throw outside();
+    const rel = containerPath.slice(mount.container.length + 1);
+    const file = path.join(mount.host, rel);
+    const back = path.relative(mount.host, file);
+    if (back === '..' || back.startsWith(`..${path.sep}`) || path.isAbsolute(back)) throw outside();
+    return { mount, rel, file };
+}
+
+// Whether a step may hand this file back: it lies below one of the step's output roots, and below a store
+// root (`registry/src/*/*`) only inside a unit the machine does not hold yet, as those never change.
+export function isDeclaredOutput(rel, outputRoots, present = []) {
+    const parts = rel.split('/');
+    return outputRoots.some((root) => {
+        if (!root.includes('*')) return root === '.' || rel === root || rel.startsWith(`${root}/`);
+        const depth = root.split('/').length;
+        const unit = parts.slice(0, depth).join('/');
+        return parts.length > depth && matchesPattern(unit, root) && !present.includes(unit);
+    });
+}