crossbind 2.0.0-beta.69 → 2.0.0-beta.72

package/src/utils/packageSourceInstall.js

@@ -0,0 +1,162 @@
+import fs from 'node:fs';
+import upath from 'upath';
+import withDirLock from './dirLock.js';
+import logger from './logger.js';
+import loadJson from './loadJson.js';
+import writeIfChanged from './writeIfChanged.js';
+import { getContentHash } from './hash.js';
+import { getDockerImage, imageRoleFor } from './pullDockerImage.js';
+import { extensionStep, isJsonValue, stepFailure } from './appExtensions.js';
+import {
+    assertRunsOnHost, assertRunsRemotely, clearSourceWork, createSourceProbe, createSourceWork, localCompilerIdentity,
+    removeSourceWork, sourceRoot, sourceRunner, toHostPath,
+} from './runPackageSource.js';
+import packageSourceApi from './packageSourceApi.js';
+import { TOOLCHAIN_PLATFORMS, targetToolchain } from './targetToolchain.js';
+import {
+    declaredSources, dependenciesKey, declarationOf, dependenciesOf,
+} from './packageSources.js';
+import {
+    installedPackages, stagePackage, writeSourceManifest, readSourceManifest,
+} from './packageSourceStage.js';
+import { sourceStageDir } from './packageSourceImport.js';
+
+// Bumped when the stage changes shape, so a stage an older crossbind made is made again.
+const STAGE_FORMAT = 4;
+const MAX_LOCK_LENGTH = 8 * 1024 * 1024;
+
+const lockFileOf = (config, name) => upath.join(config.paths.project, `${name}.lock`);
+const stampFileOf = (stageDir, target) => upath.join(stageDir, 'stamps', `${target.path}.json`);
+const logFileOf = (stageDir, target) => upath.join(stageDir, 'logs', `${target.path}.log`);
+const lockOf = (file) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null);
+// A source gets a copy: what it does to one changes nothing of crossbind's.
+const copyOf = (value) => JSON.parse(JSON.stringify(value));
+
+// What crossbind builds the target with, which a source builds its packages to match.
+function toolchainOf(name, target) {
+    const toolchain = targetToolchain(target);
+    if (toolchain) return toolchain;
+    const names = TOOLCHAIN_PLATFORMS;
+    throw new Error(`crossbind: ${name}: imports build for ${names.slice(0, -1).join(', ')} and ${names.at(-1)} so far; ${target.platform} is not supported yet, so its build cannot use ${declarationOf(name)}.`);
+}
+
+// What a source says its packages are made from beside what crossbind knows of.
+function inputsOf(source, config, target, toolchain) {
+    const { name, packageSource } = source;
+    if (!packageSource.inputs) return null;
+    const api = packageSourceApi({
+        source, config, target, work: createSourceProbe(name, sourceRunner(config, target)),
+    });
+    let inputs;
+    try {
+        inputs = packageSource.inputs(api, { dependencies: copyOf(dependenciesOf(config, name)), toolchain: copyOf(toolchain) });
+    } catch (error) {
+        throw stepFailure(name, 'inputs', error);
+    }
+    if (!isJsonValue(inputs)) throw new Error(`crossbind: ${name}'s inputs gives no JSON value, whose change is what stages its packages again.`);
+    return inputs;
+}
+
+// What a staged target was made from. The identity is the image, or the host's compiler where the tool runs on
+// the host (RUNNER=LOCAL, and iOS and macOS always): packages build with that compiler, so a new one must restage.
+function stampKey(source, config, target) {
+    const { name } = source;
+    const toolchain = toolchainOf(name, target);
+    return getContentHash(JSON.stringify({
+        format: STAGE_FORMAT,
+        source: name,
+        dependencies: dependenciesOf(config, name),
+        toolchain,
+        identity: sourceRunner(config, target) === 'LOCAL' ? localCompilerIdentity(target) : getDockerImage(imageRoleFor(target)),
+        lock: lockOf(lockFileOf(config, name)),
+        inputs: inputsOf(source, config, target, toolchain),
+    }));
+}
+
+// The target's manifest names the dependencies it was staged for.
+const isStaged = (source, config, stageDir, target) => loadJson(stampFileOf(stageDir, target))?.key === stampKey(source, config, target)
+    && readSourceManifest(source.name, stageDir, dependenciesKey(dependenciesOf(config, source.name)), target.path) !== null;
+
+// The lock comes back from where the recipes ran, and every later build reads it again. The source says whether
+// it is a lock; crossbind takes text alone, of no more than a lock gets.
+function writeLock(name, file, lock) {
+    if (lock === null || lock === undefined) return;
+    if (typeof lock !== 'string' || lock.length > MAX_LOCK_LENGTH || lock.includes('\0')) {
+        throw new Error(`crossbind: ${name}'s install gives no lock crossbind can keep, which is text of 8 MiB at most, so ${name}.lock is left as it was.`);
+    }
+    writeIfChanged(file, lock);
+}
+
+async function installTarget(source, config, stageDir, target) {
+    const { name, packageSource } = source;
+    const dependencies = dependenciesOf(config, name);
+    const lockFile = lockFileOf(config, name);
+    const stampFile = stampFileOf(stageDir, target);
+    const logFile = logFileOf(stageDir, target);
+    const runner = sourceRunner(config, target);
+    const toolchain = toolchainOf(name, target);
+    if (runner === 'LOCAL') assertRunsOnHost(name, target);
+    if (runner === 'REMOTE') assertRunsRemotely(source);
+    // A restage that stops halfway must not pass for a finished one.
+    fs.rmSync(stampFile, { force: true });
+    const work = createSourceWork(name, runner);
+    const label = `${name} ${target.path}`;
+    logger.startTask(label);
+    try {
+        const api = packageSourceApi({
+            source, config, target, work, logFile,
+        });
+        const installed = await extensionStep(name, 'install', () => packageSource.install(api, {
+            dependencies: copyOf(dependencies), toolchain: copyOf(toolchain), lock: lockOf(lockFile),
+        }));
+        const packages = installedPackages(name, installed?.packages);
+        const distCmake = fs.readFileSync(upath.join(config.paths.cli, 'assets', 'cmake', 'dist.cmake'), 'utf8');
+        packages.forEach((pkg) => stagePackage(name, pkg, {
+            stageDir, targetPath: target.path, toHost: (reported) => toHostPath(reported, work), store: work.store, distCmake,
+        }));
+        writeSourceManifest(stageDir, target.path, {
+            key: dependenciesKey(dependencies),
+            packages: packages.map(({
+                packageFolder, includedirs, libdirs, ...entry
+            }) => entry),
+        });
+        writeLock(name, lockFile, installed.lock);
+        writeIfChanged(stampFile, `${JSON.stringify({ key: stampKey(source, config, target) })}\n`);
+        logger.doneTask(label, packages.map((pkg) => `${pkg.name} ${pkg.version}`).join(', '));
+    } finally {
+        // Not over the error that ended the run; the next install clears what is left.
+        try {
+            removeSourceWork(work);
+        } catch (e) {
+            logger.info(`crossbind: could not remove ${work.dir} (${e.message}).`);
+        }
+    }
+}
+
+async function installSource(source, config, targets) {
+    const { name } = source;
+    const stageDir = sourceStageDir(config.paths.cache, name);
+    const pending = targets.filter((target) => !isStaged(source, config, stageDir, target));
+    if (pending.length === 0) return false;
+    for (const runner of new Set(pending.map((target) => sourceRunner(config, target)))) {
+        // A store takes one writer at a time, and every project on the machine shares it.
+        await withDirLock(`${sourceRoot(name, runner)}-install.lock`, async () => {
+            clearSourceWork(name, runner);
+            for (const target of pending.filter((candidate) => sourceRunner(config, candidate) === runner)) {
+                // Another build may have staged it while this one waited for the store.
+                if (!isStaged(source, config, stageDir, target)) await installTarget(source, config, stageDir, target);
+            }
+        });
+    }
+    return true;
+}
+
+// Installs the packages a project declares of each source for every target not staged from the same inputs
+// yet, and says whether it staged any, so the caller attaches the new manifests.
+export default async function installPackageSources(config, targets) {
+    let hasStaged = false;
+    for (const source of declaredSources(config)) {
+        hasStaged = await installSource(source, config, targets) || hasStaged;
+    }
+    return hasStaged;
+}

package/src/utils/{conanStage.js → packageSourceStage.js}

@@ -6,10 +6,11 @@ import writeIfChanged from './writeIfChanged.js';
 import makeTreeWritable from './makeTreeWritable.js';
 import logger from './logger.js';
 import { libNameOf } from './linkLayout.js';
-import { CONAN_NAME, conanPackageDir } from './conanImport.js';
-import { CONAN_VERSION } from './conanDependencies.js';
+import { PACKAGE_NAME, sourcePackageDir } from './packageSourceImport.js';
+import { isControlCharacter, printable } from './printableText.js';
 
 const MANIFESTS = 'manifests';
+const PACKAGE_VERSION = /^[A-Za-z0-9_][A-Za-z0-9_.+-]*$/;
 const LIB_NAME = /^[A-Za-z0-9_][A-Za-z0-9_.+-]*$/;
 // A macro name with a plain value, if any (a path or a list too): one compiler argument, in a list CMake
 // splits on ';'.
@@ -20,17 +21,16 @@ const URL = /^https?:\/\/[^\s"'<>`\\]+$/;
 const SHA256 = /^[0-9a-f]{64}$/;
 const MAX_TEXT = 1000;
 const TARGET_PATH = /^[A-Za-z0-9_-]+$/;
-// The prefix of the variables a package's CMakeLists names each target's libraries by.
-const HOST_LIBS = 'CROSSBIND_CONAN_LIBS_';
 
-// The CMake target of a package; Conan names may carry '.', '+' and '-'.
-export const conanTargetName = (name) => `conan_${name.replace(/[^A-Za-z0-9_]/g, '_')}`;
+// The CMake target of a package; package names may carry '.', '+' and '-'.
+export const sourceTargetName = (source, name) => `${source}_${name.replace(/[^A-Za-z0-9_]/g, '_')}`;
+// The prefix of the variables a package's CMakeLists names each target's libraries by.
+const hostLibsOf = (source) => `CROSSBIND_${source.toUpperCase()}_LIBS_`;
 
 const matches = (pattern, value) => typeof value === 'string' && pattern.test(value);
 const isListOf = (pattern, values) => Array.isArray(values) && values.every((value) => matches(pattern, value));
-const shown = (value) => `${JSON.stringify(value)}`.slice(0, 200);
-// C0 and C1 controls, which carry terminal escape sequences.
-export const isControlCharacter = (char) => char < ' ' || (char >= '\u007f' && char <= '\u009f');
+// What a recipe wrote, as a message quotes it: JSON escapes the C0 controls, and the others go.
+const shown = (value) => printable(`${JSON.stringify(value)}`).slice(0, 200);
 // Recipe text ends up in notices and terminals: one line, no control characters.
 const textOf = (value) => (typeof value === 'string' && value.length <= MAX_TEXT && ![...value].some(isControlCharacter) ? value : null);
 const urlOf = (value) => (matches(URL, value) && value.length <= MAX_TEXT ? value : null);
@@ -42,19 +42,19 @@ function licenseOf(license) {
     return parts.length === 1 ? parts[0] : parts.map((part) => (part.includes(' OR ') ? `(${part})` : part)).join(' AND ');
 }
 
-// The fields of a package that crossbind keeps. Conan's JSON comes from recipe code and these fields
-// end up in paths, CMake files and notices, so they are checked where they come in and again where
+// The fields of a package that crossbind keeps. What a source reports comes from recipe code and these
+// fields end up in paths, CMake files and notices, so they are checked where they come in and again where
 // the manifest is read back.
-export function conanPackageEntry(entry) {
+export function packageEntry(source, entry) {
     const fail = (field) => {
-        throw new Error(`crossbind: the Conan package ${shown(entry.ref ?? entry.name)} has an invalid ${field}: ${shown(entry[field])}.`);
+        throw new Error(`crossbind: the ${source} package ${shown(entry.ref ?? entry.name)} has an invalid ${field}: ${shown(entry[field])}.`);
     };
-    if (!matches(CONAN_NAME, entry.name)) fail('name');
-    if (!matches(CONAN_VERSION, entry.version)) fail('version');
+    if (!matches(PACKAGE_NAME, entry.name)) fail('name');
+    if (!matches(PACKAGE_VERSION, entry.version)) fail('version');
     const reference = `${entry.name}/${entry.version}`;
     if (typeof entry.ref !== 'string' || !entry.ref.startsWith(reference) || !REFERENCE_TAIL.test(entry.ref.slice(reference.length))) fail('ref');
     if (!isListOf(LIB_NAME, entry.libs)) fail('libs');
-    if (!isListOf(CONAN_NAME, entry.requires)) fail('requires');
+    if (!isListOf(PACKAGE_NAME, entry.requires)) fail('requires');
     // They become linker and compiler arguments. A manifest an older crossbind wrote has none of them.
     const systemLibs = entry.systemLibs ?? [];
     const frameworks = entry.frameworks ?? [];
@@ -78,50 +78,30 @@ export function conanPackageEntry(entry) {
     };
 }
 
-function sourceOf(entry) {
-    const source = Array.isArray(entry) ? entry[0] : entry;
-    if (!source?.url) return null;
-    return { url: Array.isArray(source.url) ? source.url[0] : source.url, sha256: source.sha256 ?? null };
-}
-
 // A define crossbind cannot pass as one plain argument was left out before crossbind took defines at all,
 // so it stays out, with a notice, instead of stopping the build.
-function plainDefinesOf(node, defines) {
-    return defines.filter((define) => {
+function plainDefinesOf(pkg) {
+    if (!Array.isArray(pkg.defines)) return pkg.defines;
+    return pkg.defines.filter((define) => {
         if (matches(DEFINE, define)) return true;
-        logger.info(`crossbind: ${shown(node.ref)} asks the code that uses it to compile with ${shown(define)}, which crossbind leaves out: a define passes as a plain name with a plain value.`);
+        logger.info(`crossbind: ${shown(pkg.ref)} asks the code that uses it to compile with ${shown(define)}, which crossbind leaves out: a define passes as a plain name with a plain value.`);
         return false;
     });
 }
 
-function packageOf(node, requires) {
-    const components = Object.values(node.cpp_info ?? {});
-    const collect = (key) => [...new Set(components.flatMap((component) => component?.[key] ?? []))];
+function installedPackage(source, pkg) {
     const folders = (key) => {
-        const values = collect(key);
-        if (!values.every((value) => typeof value === 'string')) {
-            throw new Error(`crossbind: the Conan package ${shown(node.ref)} has invalid ${key}.`);
+        if (!Array.isArray(pkg[key]) || !pkg[key].every((value) => typeof value === 'string')) {
+            throw new Error(`crossbind: the ${source} package ${shown(pkg.ref)} has invalid ${key}.`);
         }
-        return values;
+        return [...pkg[key]];
     };
-    if (typeof node.package_folder !== 'string') {
-        throw new Error(`crossbind: conan reported the package ${shown(node.ref)} without a package folder.`);
+    if (typeof pkg.packageFolder !== 'string') {
+        throw new Error(`crossbind: ${source} reported the package ${shown(pkg.ref)} without a package folder.`);
     }
     return {
-        ...conanPackageEntry({
-            name: node.name,
-            version: node.version,
-            ref: node.ref,
-            license: node.license,
-            homepage: node.homepage,
-            source: sourceOf(node.conandata?.sources?.[node.version]),
-            libs: collect('libs'),
-            requires,
-            systemLibs: collect('system_libs'),
-            frameworks: collect('frameworks'),
-            defines: plainDefinesOf(node, collect('defines')),
-        }),
-        packageFolder: node.package_folder,
+        ...packageEntry(source, { ...pkg, defines: plainDefinesOf(pkg) }),
+        packageFolder: pkg.packageFolder,
         includedirs: folders('includedirs'),
         libdirs: folders('libdirs'),
     };
@@ -143,30 +123,22 @@ function linkOrder(packages) {
     return order.reverse();
 }
 
-// Host packages of a `conan install --format=json` graph in link order. Left out are the consumer,
-// build tools, test requirements and packages conan skipped because the consumer needs nothing of them.
-export function conanPackagesOf(graph) {
-    const nodes = graph?.nodes;
-    if (!nodes || typeof nodes !== 'object' || !graph.root) {
-        throw new Error('crossbind: conan install printed a graph without nodes or a root.');
+// The packages a source's install reports, checked, in link order, with the folders the source found them in.
+export function installedPackages(source, packages) {
+    if (!Array.isArray(packages) || !packages.every((pkg) => pkg !== null && typeof pkg === 'object')) {
+        throw new Error(`crossbind: ${source}'s install gives no list of the packages it installed.`);
     }
-    const [root] = Object.keys(graph.root);
-    const isPackage = (id) => id !== root && nodes[id]?.context === 'host' && !nodes[id].test && nodes[id].binary !== 'Skip';
-    const ids = Object.keys(nodes).filter(isPackage);
-    const requiresOf = (id) => Object.entries(nodes[id].dependencies ?? {})
-        .filter(([dependency, edge]) => !edge?.build && !edge?.test && ids.includes(dependency))
-        .map(([dependency]) => nodes[dependency].name);
-    return linkOrder(ids.map((id) => packageOf(nodes[id], requiresOf(id))));
+    return linkOrder(packages.map((pkg) => installedPackage(source, pkg)));
 }
 
-// The real path of a file conan reported, or null when there is none; refused when it resolves
+// The real path of a file a source reported, or null when there is none; refused when it resolves
 // outside `root`.
-function realPathWithin(file, root, pkg) {
+function realPathWithin(source, file, root, pkg) {
     if (!fs.existsSync(file)) return null;
     const real = fs.realpathSync(file);
     const relative = path.relative(root, real);
     if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
-        throw new Error(`crossbind: the Conan package ${pkg.ref} reports ${file}, which resolves outside ${root}.`);
+        throw new Error(`crossbind: the ${source} package ${pkg.ref} reports ${printable(file)}, which resolves outside ${root}.`);
     }
     return real;
 }
@@ -175,15 +147,15 @@ function realPathWithin(file, root, pkg) {
 // directories and regular files inside the package, and into each linked directory once: a link out
 // of the package could point the copy at any file the host can read, and links back up the tree
 // would copy it without end.
-function copyTree(from, to, packageRoot, pkg) {
+function copyTree(from, to, within) {
     const linked = new Set([from]);
     fs.cpSync(from, to, {
         recursive: true,
         dereference: true,
-        filter: (source) => {
-            const entry = fs.lstatSync(source);
+        filter: (file) => {
+            const entry = fs.lstatSync(file);
             if (!entry.isSymbolicLink()) return entry.isDirectory() || entry.isFile();
-            const real = realPathWithin(source, packageRoot, pkg);
+            const real = within(file);
             const target = real && fs.statSync(real);
             if (!target?.isDirectory()) return Boolean(target?.isFile());
             if (linked.has(real)) return false;
@@ -196,10 +168,10 @@ function copyTree(from, to, packageRoot, pkg) {
 
 // The package's own folder: no link, and below the store's own layout, so it cannot take in the
 // store or another package.
-function packageRootOf(pkg, folder, store) {
-    const real = realPathWithin(folder, store, pkg);
+function packageRootOf(source, pkg, folder, store) {
+    const real = realPathWithin(source, folder, store, pkg);
     if (!real || fs.lstatSync(folder).isSymbolicLink() || path.relative(store, real).split(path.sep).length < 2) {
-        throw new Error(`crossbind: the Conan package ${pkg.ref} has no package folder crossbind can use at ${pkg.packageFolder}.`);
+        throw new Error(`crossbind: the ${source} package ${pkg.ref} has no package folder crossbind can use at ${printable(pkg.packageFolder)}.`);
     }
     return real;
 }
@@ -207,35 +179,36 @@ function packageRootOf(pkg, folder, store) {
 // A package lands where a published port keeps its prebuilt (<dir>/dist/prebuilt/<target>/{include,lib}),
 // so the dependency machinery links it and SWIG treats its headers as one across targets. Everything
 // is read from inside the package's own folder in the store.
-export function stageConanPackage(pkg, {
+export function stagePackage(source, pkg, {
     stageDir, targetPath, toHost, store, distCmake,
 }) {
-    const packageRoot = packageRootOf(pkg, toHost(pkg.packageFolder), fs.realpathSync(store));
-    const reported = (folder) => realPathWithin(toHost(upath.isAbsolute(folder) ? folder : upath.join(pkg.packageFolder, folder)), packageRoot, pkg);
-    const dir = conanPackageDir(stageDir, pkg.name);
+    const packageRoot = packageRootOf(source, pkg, toHost(pkg.packageFolder), fs.realpathSync(store));
+    const within = (file) => realPathWithin(source, file, packageRoot, pkg);
+    const reported = (folder) => within(toHost(upath.isAbsolute(folder) ? folder : upath.join(pkg.packageFolder, folder)));
+    const dir = sourcePackageDir(stageDir, pkg.name);
     const prebuilt = upath.join(dir, 'dist', 'prebuilt');
     const prefix = upath.join(prebuilt, targetPath);
 
     fs.rmSync(prefix, { recursive: true, force: true });
     fs.mkdirSync(upath.join(prefix, 'include'), { recursive: true });
     fs.mkdirSync(upath.join(prefix, 'lib'), { recursive: true });
-    pkg.includedirs.map(reported).filter(Boolean).forEach((folder) => copyTree(folder, upath.join(prefix, 'include'), packageRoot, pkg));
+    pkg.includedirs.map(reported).filter(Boolean).forEach((folder) => copyTree(folder, upath.join(prefix, 'include'), within));
     const libdirs = pkg.libdirs.map(reported).filter(Boolean);
     pkg.libs.forEach((lib) => {
-        const archive = libdirs.map((folder) => realPathWithin(path.join(folder, `lib${lib}.a`), packageRoot, pkg))
+        const archive = libdirs.map((folder) => within(path.join(folder, `lib${lib}.a`)))
             .find((file) => file && fs.statSync(file).isFile());
         if (!archive) {
-            throw new Error(`crossbind: the Conan package ${pkg.ref} declares the library ${lib}, but its package has no lib${lib}.a.`);
+            throw new Error(`crossbind: the ${source} package ${pkg.ref} declares the library ${lib}, but its package has no lib${lib}.a.`);
         }
         fs.copyFileSync(archive, upath.join(prefix, 'lib', `lib${lib}.a`));
     });
 
     const licenses = upath.join(dir, 'licenses');
     fs.rmSync(licenses, { recursive: true, force: true });
-    const shipped = realPathWithin(path.join(packageRoot, 'licenses'), packageRoot, pkg);
-    if (shipped && fs.statSync(shipped).isDirectory()) copyTree(shipped, licenses, packageRoot, pkg);
+    const shipped = within(path.join(packageRoot, 'licenses'));
+    if (shipped && fs.statSync(shipped).isDirectory()) copyTree(shipped, licenses, within);
     writeIfChanged(upath.join(dir, 'package.json'), `${JSON.stringify({
-        name: `conan:${pkg.name}`, version: pkg.version, nativeVersion: pkg.version, license: pkg.license, private: true,
+        name: `${source}:${pkg.name}`, version: pkg.version, nativeVersion: pkg.version, license: pkg.license, private: true,
     }, null, 4)}\n`);
 
     const hosts = fs.readdirSync(prebuilt, { withFileTypes: true })
@@ -247,12 +220,13 @@ export function stageConanPackage(pkg, {
         if (!fs.existsSync(libdir)) return [];
         return fs.readdirSync(libdir).filter((file) => file.endsWith('.a')).map(libNameOf).filter((name) => matches(LIB_NAME, name)).sort();
     };
+    const hostLibs = hostLibsOf(source);
     writeIfChanged(upath.join(prebuilt, 'CMakeLists.txt'), [
-        ...hosts.map((host) => `set(${HOST_LIBS}${host} "${archivesOf(host).join(';')}")`),
+        ...hosts.map((host) => `set(${hostLibs}${host} "${archivesOf(host).join(';')}")`),
         distCmake
-            .replace('___PROJECT_NAME___', () => conanTargetName(pkg.name))
+            .replace('___PROJECT_NAME___', () => sourceTargetName(source, pkg.name))
             .replace('___PROJECT_HOST___', () => hosts.join(';'))
-            .replace('___PROJECT_LIBS___', () => `\${${HOST_LIBS}\${PROJECT_TARGET_HOST}}`)
+            .replace('___PROJECT_LIBS___', () => `\${${hostLibs}\${PROJECT_TARGET_HOST}}`)
             .replace('___PROJECT_WHOLE_ARCHIVE___', () => ''),
     ].join('\n'));
 }
@@ -261,14 +235,14 @@ const manifestsOf = (stageDir) => upath.join(stageDir, MANIFESTS);
 const manifestFileOf = (stageDir, targetPath) => upath.join(manifestsOf(stageDir), `${targetPath}.json`);
 
 // One manifest per target: a recipe can require other packages on another platform.
-export function writeConanManifest(stageDir, targetPath, manifest) {
+export function writeSourceManifest(stageDir, targetPath, manifest) {
     writeIfChanged(manifestFileOf(stageDir, targetPath), `${JSON.stringify(manifest, null, 4)}\n`);
 }
 
 // The packages staged for one target, or for every target staged so far, in link order, and each
-// target's own list. A manifest staged for other conanDependencies describes packages the config no
-// longer asks for.
-export function readConanManifest(stageDir, key, targetPath = null) {
+// target's own list. A manifest staged for other dependencies describes packages the config no longer
+// asks for.
+export function readSourceManifest(source, stageDir, key, targetPath = null) {
     const dir = manifestsOf(stageDir);
     let files = [];
     if (targetPath) files = [manifestFileOf(stageDir, targetPath)];
@@ -277,7 +251,7 @@ export function readConanManifest(stageDir, key, targetPath = null) {
         .filter(({ manifest }) => manifest?.key === key);
     if (manifests.length === 0) return null;
     try {
-        const targets = Object.fromEntries(manifests.map(({ target, manifest }) => [target, manifest.packages.map(conanPackageEntry)]));
+        const targets = Object.fromEntries(manifests.map(({ target, manifest }) => [target, manifest.packages.map((entry) => packageEntry(source, entry))]));
         return { key, packages: linkOrder(Object.values(targets).flat()), targets };
     } catch (e) {
         throw new Error(`crossbind: ${dir} lists a package crossbind will not use - delete ${stageDir} and build again. ${e.message}`, { cause: e });
@@ -285,7 +259,7 @@ export function readConanManifest(stageDir, key, targetPath = null) {
 }
 
 // Changes whenever a build stages packages, so a process that attached before can tell.
-export function conanManifestsStamp(stageDir) {
+export function sourceManifestsStamp(stageDir) {
     const dir = manifestsOf(stageDir);
     if (!fs.existsSync(dir)) return '';
     return fs.readdirSync(dir).sort().map((name) => `${name}@${fs.statSync(upath.join(dir, name)).mtimeMs}`).join('|');

package/src/utils/packageSources.js

@@ -0,0 +1,108 @@
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import {
+    broughtExtension, isJsonValue, ownPackages, stepFailure, withListed,
+} from './appExtensions.js';
+import { getContentHash } from './hash.js';
+import { PACKAGE_NAME, declarationOf, dependenciesOf } from './packageSourceImport.js';
+
+// crossbind's own folder, whose dependencies name the extensions it brings.
+const CORE_ROOT = path.join(import.meta.dirname, '..', '..');
+
+// The extension packages crossbind itself brings, which a project neither installs nor lists.
+export const packagesOfCrossbind = () => ownPackages(CORE_ROOT);
+
+let own = null;
+
+// crossbind's own package sources: those the extension packages among its dependencies give, which every project
+// has by having crossbind. Loaded once, as a source may keep what it asked its tool.
+function ownSources() {
+    if (!own) {
+        const load = createRequire(path.join(CORE_ROOT, 'package.json'));
+        own = packagesOfCrossbind()
+            .map((name) => broughtExtension(load, name, `update ${name} to match crossbind`))
+            .filter((extension) => extension.packageSource);
+    }
+    return own;
+}
+
+// A project's package sources: crossbind's own, then those crossbind.config.js lists. One listed under the name of
+// one of crossbind's takes its place.
+export const packageSourcesOf = (listed) => withListed(ownSources(), listed.filter((extension) => extension.packageSource));
+
+export { declarationOf, dependenciesOf };
+
+// The sources a project declares packages of.
+export const declaredSources = (config) => (config.packageSources ?? []).filter(({ name }) => Object.keys(dependenciesOf(config, name)).length > 0);
+
+export const sourceNamesOf = (config) => (config.packageSources ?? []).map(({ name }) => name);
+
+// What an import names a store by, as the node: and npm: schemes do: cargo's crates, and each source's packages.
+export const importSchemesOf = (config) => ['cargo', ...sourceNamesOf(config)];
+
+// What a project declares for a source, as the source's check takes it: a map of package names to what each asks
+// for, in plain JSON, as it is hashed, kept in manifests and handed back to the source.
+export function checkedDependencies({ name, packageSource }, raw) {
+    let dependencies;
+    try {
+        dependencies = packageSource.check(raw);
+    } catch (error) {
+        throw stepFailure(name, 'check', error);
+    }
+    const isMap = dependencies !== null && typeof dependencies === 'object' && !Array.isArray(dependencies) && isJsonValue(dependencies);
+    if (!isMap || !Object.keys(dependencies).every((pkg) => PACKAGE_NAME.test(pkg))) {
+        throw new Error(`crossbind: ${name}'s check gives no ${declarationOf(name)} crossbind can use: a JSON map of package names, of lower-case letters, digits and _ + . -, to what each asks for.`);
+    }
+    return dependencies;
+}
+
+// cargo's crates, each as Cargo.toml takes it: a version, or the table that would follow `<crate> =`. Text, where a
+// source's package may be an object: a number would lose a trailing zero, 0.10 becoming 0.1.
+function checkedCrates(crates = {}) {
+    if (crates === null || typeof crates !== 'object' || Array.isArray(crates)) {
+        throw new Error(`crossbind: ${declarationOf('cargo')} must be an object of crates, such as { uuid: '1' }.`);
+    }
+    const [crate] = Object.entries(crates).find(([, spec]) => typeof spec !== 'string') ?? [];
+    if (crate !== undefined) {
+        throw new Error(`crossbind: ${declarationOf('cargo')}.${crate} must be text as Cargo.toml takes it: a version such as '1', or a table such as '{ version = "1", features = ["v4"] }'.`);
+    }
+    return { ...crates };
+}
+
+// What a project declares under packages, by the name its imports open with: cargo's crates, which are the core's
+// own, and each source's packages. Another name stops the config: a misspelt one would be left unread. So does the
+// key cargo or a source once had of its own, with where its packages are declared now and the config that holds it:
+// a package crossbind builds from source loads as a project too, and its key is not the app's developer's to find.
+export function checkedPackages(sources, config, file = 'crossbind.config') {
+    const names = sources.map(({ name }) => name);
+    const moved = ['cargo', ...names].find((name) => config[`${name}Dependencies`] !== undefined);
+    if (moved) throw new Error(`crossbind: ${moved}Dependencies is now ${declarationOf(moved)}: write packages: { ${moved}: { … } } in ${file}.`);
+    const { packages = {} } = config;
+    if (packages === null || typeof packages !== 'object' || Array.isArray(packages)) {
+        throw new Error("crossbind: \"packages\" must be an object of each package source's packages, such as { conan: { zlib: '1.3.2' } }.");
+    }
+    const unknown = Object.keys(packages).find((name) => name !== 'cargo' && !names.includes(name));
+    if (unknown !== undefined) {
+        throw new Error(`crossbind: ${declarationOf(unknown)} names neither cargo nor a package source of this project, which has ${names.join(', ')}. List its extension in extensions, or fix the name.`);
+    }
+    const written = (name) => (Object.hasOwn(packages, name) ? packages[name] : undefined);
+    return {
+        cargo: checkedCrates(written('cargo')),
+        ...Object.fromEntries(sources.map((source) => [source.name, checkedDependencies(source, written(source.name))])),
+    };
+}
+
+// What staged packages were installed for; a change to the dependencies makes them stale.
+export const dependenciesKey = (dependencies) => getContentHash(JSON.stringify(dependencies));
+
+// What an app's headers and sources see of its sources' packages: the packages are staged after a header may
+// already be bound, and restaged in place when one moves to another version. Null without any.
+export function packageSourceInputsOf(config) {
+    const declared = declaredSources(config);
+    if (declared.length === 0) return null;
+    return declared.map(({ name }) => ({
+        source: name,
+        dependencies: dependenciesOf(config, name),
+        refs: config.allDependencies.filter(({ general }) => general.packageSource?.name === name).map(({ general }) => general.packageSource.package.ref),
+    }));
+}

package/src/utils/printableText.js

@@ -0,0 +1,5 @@
+// C0 and C1 controls, which carry terminal escape sequences.
+export const isControlCharacter = (char) => char < ' ' || (char >= '\u007f' && char <= '\u009f');
+
+// Text a recipe or a tool wrote, for a terminal or a message: its lines and tabs, without any other control character.
+export const printable = (text) => [...text].filter((char) => char === '\n' || char === '\t' || !isControlCharacter(char)).join('');