crossbind 2.0.0-beta.62 → 2.0.0-beta.64

package/src/integration/getCrossbindScript.js

@@ -1,22 +1,57 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import getData from '../actions/getData.js';
+import getEnv from '../actions/getEnv.js';
 import loadJson from '../utils/loadJson.js';
 import state from '../state/index.js';
 import { parseSurface, createRustBridgeCrate, createCrateImportBridge } from '../utils/rustBridgeGen.js';
 import { parseCargoMarkerName } from '../utils/cargoImport.js';
+import { UNBOUND_MESSAGE } from '../assets/js-runtime/adapters/unboundMessage.js';
 
-export default function getCrossbindScript(target, bridgePath) {
+// `base` is where the bundler serves the app from, as Vite's `base` names it; webpack's public path wins at run time.
+// Metro asks for `liveExports`: it keeps a header's module as it first transformed it while the app's imports change.
+export default function getCrossbindScript(target, bridgePath, { base = '/', liveExports: isLive = false } = {}) {
     if (!bridgePath) {
         throw new Error('getCrossbindScript needs the bridge file of the imported header');
     }
-    return buildScript(target, loadJson(`${bridgePath}.exports.json`));
+    const script = buildScript(target, loadJson(`${bridgePath}.exports.json`), base);
+    return isLive ? `${script}${liveExportsScript()}` : script;
+}
+
+// A name the module does not export comes off the booted module: undefined before initNative. A name the module the page
+// or app loaded lacks, exported or not, is a function that says why. Serialized into the generated module, so it
+// references nothing outside itself.
+export function liveExports(exportsObject, unbound) {
+    return new Proxy(exportsObject, {
+        get(target, name, receiver) {
+            const value = Reflect.get(target, name, receiver);
+            const bound = globalThis.__crossbindModule;
+            if (value != null || !bound || typeof name !== 'string' || ['then', 'default', 'toJSON'].includes(name)) return value;
+            if (bound[name] !== undefined) return bound[name];
+            if (!/^[A-Za-z_]\w*$/.test(name)) return value;
+            return function notBound() {
+                throw new Error(`crossbind: ${name} ${unbound}`);
+            };
+        },
+    });
+}
+
+function liveExportsScript() {
+    return `
+        const __crossbindLiveExports = ${liveExports.toString()};
+        if (typeof module === 'object' && module) module.exports = __crossbindLiveExports(module.exports, ${JSON.stringify(UNBOUND_MESSAGE)});
+    `;
 }
 
 // The Rust analog of a .h import: parse the crate surface and emit the same proxy module
 // (per-symbol lets assigned inside this module's initNative). Vectors come from the owning
 // cargo package's config; classes/enums come from the parsed source.
-export function getRustJsScript(target, rsFile) {
+export function getRustJsScript(target, rsFile, { base = '/' } = {}) {
+    return buildScript(target, getRustSymbols(rsFile), base);
+}
+
+// The names a Rust import exports. An app-local file or a cargo: crate gets its bridge crate here, the way a header
+// gets its bridge from createBridgeFile.
+export function getRustSymbols(rsFile) {
     // Compare real paths: dependency paths go through node_modules symlinks (pnpm workspaces)
     // while bundlers hand the transformer the resolved real file.
     const realCrateDir = (d) => {
@@ -35,12 +70,13 @@ export function getRustJsScript(target, rsFile) {
             modulePath,
             spec: cargoDeps[crateName],
             cacheDir: state.config.paths.cache,
+            projectDir: state.config.paths.project,
             dtsMode: state.config.dts,
             log: () => {},
         });
         // A crate registers its public names under per-crate names (two crates may export the
         // same name), so the proxy exports the clean name and reads the registered one off the module.
-        return buildScript(target, exports);
+        return exports;
     }
 
     const pkg = state.config.allDependencies.find((d) => d.export?.type === 'cargo'
@@ -60,7 +96,7 @@ export function getRustJsScript(target, rsFile) {
             log: () => {},
         });
     }
-    const symbols = [
+    return [
         ...model.classes.map((c) => c.name),
         ...(model.streams ?? []).map((s) => s.name),
         ...model.enums.map((e) => e.name),
@@ -69,14 +105,13 @@ export function getRustJsScript(target, rsFile) {
         ...(model.consts ?? []).map((c) => c.name),
         ...vectors.map((v) => v.name),
     ];
-    return buildScript(target, symbols);
 }
 
-function buildScript(target, symbols) {
+function buildScript(target, symbols, base) {
     if (!target) {
         throw new Error('The target is not available!');
     }
-    const env = JSON.stringify(getData('env', target));
+    const env = JSON.stringify(getEnv(target));
     const getPlatformScript = target.platform === 'wasm' ? getWebScript : getReactNativeScript;
 
     let symbolExportDefineString = '';
@@ -90,7 +125,7 @@ function buildScript(target, symbols) {
     }
 
     return `
-        ${getPlatformScript(env)}
+        ${getPlatformScript(env, base)}
 
         export let AllSymbols = {};
         ${symbolExportDefineString}
@@ -144,7 +179,7 @@ function getReactNativeScript(env) {
         const RNJsiLib = TurboModuleRegistry.get('RNJsiLib');
 
         function setEnv() {
-            const env = JSON.parse('${env}');
+            const env = ${env};
             const CROSSBIND_DATA_PATH = Module.Crossbind.getEnv('CROSSBIND_DATA_PATH');
 
             Object.entries(env).forEach(([key, value]) => {
@@ -165,8 +200,9 @@ function getReactNativeScript(env) {
     `;
 }
 
-function getWebScript(env) {
+function getWebScript(env, base) {
     const params = `{
+        path: base,
         ...config,
         env: {...${env}, ...config.env},
         paths: {
@@ -179,7 +215,8 @@ function getWebScript(env) {
 
     return `
         function __crossbindBoot(config) {
-            return import(/* webpackIgnore: true */ '/crossbind.js')
+            const base = new URL((typeof __webpack_public_path__ === 'string' ? __webpack_public_path__ : ${JSON.stringify(base)}) || './', document.baseURI).href;
+            return import(/* webpackIgnore: true */ /* @vite-ignore */ base + 'crossbind.js')
                 .then(n => window.Crossbind.initNative(${params}));
         }
     `;

package/src/integration/getDependFilePath.js

@@ -2,6 +2,8 @@ import fs from 'node:fs';
 import path from 'node:path';
 import state from '../state/index.js';
 import { parseCargoImport, writeCargoMarker } from '../utils/cargoImport.js';
+import { CONAN_PREFIX, resolveConanHeader } from '../utils/conanImport.js';
+import refreshConanDependencies from '../state/refreshConanDependencies.js';
 
 const LAYOUT = { header: 'include', module: 'swig', source: '' };
 
@@ -92,6 +94,13 @@ export default function getDependFilePath(source, target) {
         return marker;
     }
 
+    // `conan:<package>/<header>` names a header of a declared Conan package, staged per target like a
+    // port's prebuilt; from there the header takes the same path as any other.
+    if (source.startsWith(CONAN_PREFIX)) {
+        refreshConanDependencies(state.config);
+        return resolveConanHeader(state.config, source, target ?? state.targets.find((t) => t.platform === 'wasm'));
+    }
+
     const matches = getMatchingPackages(source);
     if (matches.length === 0) return null;
 

package/src/integration/resolveNativeImport.js

@@ -0,0 +1,14 @@
+import { createRequire } from 'node:module';
+import upath from 'upath';
+import getDependFilePath from './getDependFilePath.js';
+
+// A package that is not a crossbind dependency (the conformance kit) resolves the way the bundler finds it.
+export default function resolveNativeImport(specifier, importer, target) {
+    if (specifier.startsWith('.')) return upath.resolve(upath.dirname(importer), specifier);
+    if (upath.isAbsolute(specifier)) return specifier;
+    try {
+        return getDependFilePath(specifier, target) ?? createRequire(importer).resolve(specifier);
+    } catch (e) {
+        return null;
+    }
+}

package/src/node/dev.mjs

@@ -0,0 +1,19 @@
+// `node --import crossbind/node/dev app.mjs`: builds this machine's binary when the app's native sources or imports
+// changed since the last build it ran, then serves the app's native imports through the hooks that build wrote.
+import { pathToFileURL } from 'node:url';
+import loadConfig from '../state/loadConfig.js';
+import { DEV_BUILD_ENV, ensureDevBuild, projectDirOf } from '../utils/nodeDevBuild.js';
+
+if (!process.env[DEV_BUILD_ENV]) {
+    // node --watch restarts when a build rewrites the outputs the last start loaded. Its SIGTERM waits for the build
+    // and its record, then ends this start: the next one finds the build done instead of starting it over.
+    let restarting = false;
+    const restart = () => { restarting = true; };
+    process.on('SIGTERM', restart);
+    const hooks = ensureDevBuild(await loadConfig(projectDirOf(process.argv[1])));
+    // A signal that came during the build is handled on the next turn of the event loop.
+    await new Promise((resolve) => { setImmediate(resolve); });
+    process.off('SIGTERM', restart);
+    if (restarting) process.exit(143);
+    if (hooks) await import(pathToFileURL(hooks).href);
+}

package/src/program.js

@@ -0,0 +1,169 @@
+import fs from 'node:fs';
+import { Command, Option } from 'commander';
+
+import {
+    deleteAccount as deleteCloudAccount, fetchUsage, login as loginToCloud, logout as logoutOfCloud, usageLines,
+} from './actions/cloudCommands.js';
+import { cloudUrl } from './utils/cloudAccount.js';
+import flattenConfigForTable from './utils/flattenConfigForTable.js';
+import loadSystemConfig, { systemConfigPath } from './utils/systemConfig.js';
+import systemKeys from './utils/systemKeys.js';
+import writeJson from './utils/writeJson.js';
+
+const packageJSON = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url)));
+
+// Build failures bubble up as exceptions with a `cause` chain (see actions/run.js);
+// show the operator a single line — DEBUG=1 for the full stack — and exit non-zero.
+export const handleFatal = (e) => {
+    if (process.env.DEBUG) {
+        console.error(e);
+    } else {
+        let message = e?.message || String(e);
+        if (e?.cause?.message) message += `\n  caused by: ${e.cause.message.split('\n')[0]}`;
+        console.error(message);
+    }
+    process.exit(1);
+};
+
+export const createProgram = () => new Command()
+    .name('crossbind')
+    .description('Compile C++ files to WebAssembly and native platforms.')
+    .version(packageJSON.version)
+    .showHelpAfterError();
+
+export function addSystemCommands(program) {
+    program.command('login')
+        .description('sign in to crossbind cloud with your GitHub account')
+        .action(async () => {
+            const url = cloudUrl(loadSystemConfig());
+            const { login } = await loginToCloud({ url });
+            console.log(`crossbind: signed in to ${url} as ${login}.`);
+        });
+
+    program.command('logout')
+        .description('sign out of crossbind cloud on this machine')
+        .option('--all', 'sign out every machine signed in to this account')
+        .action(async (options) => {
+            const url = cloudUrl(loadSystemConfig());
+            const result = await logoutOfCloud({ url, all: Boolean(options.all) });
+            if (!result) {
+                console.log(`crossbind: not signed in to ${url}.`);
+            } else if (result.revoked) {
+                console.log(`crossbind: signed ${result.login} out${options.all ? ' on every machine' : ''}.`);
+            } else {
+                console.warn(`crossbind: signed out here, but crossbind cloud did not revoke the token (${result.reason.replace(/^crossbind: /, '')}). Sign in again and run crossbind logout --all to revoke it.`);
+            }
+        });
+
+    program.command('usage')
+        .description('show what you used of crossbind cloud this month and what is left')
+        .action(async () => {
+            usageLines(await fetchUsage({ url: cloudUrl(loadSystemConfig()) })).forEach((line) => console.log(line));
+        });
+
+    program.command('account')
+        .description('manage your crossbind cloud account')
+        .command('delete')
+        .description('delete your crossbind cloud account, with the sign-in of every machine (https://crossbind.dev/privacy/)')
+        .option('--yes', 'delete it; without this, nothing is deleted')
+        .action(async (options) => {
+            const url = cloudUrl(loadSystemConfig());
+            if (!options.yes) {
+                throw new Error(`crossbind: this deletes your crossbind cloud account at ${url}, with the sign-in of every machine. Run crossbind account delete --yes to go ahead.`);
+            }
+            const { login } = await deleteCloudAccount({ url });
+            console.log(`crossbind: deleted the crossbind cloud account of ${login}.`);
+        });
+
+    const commandConfig = program.command('config')
+        .description('manage the crossbind configuration files');
+
+    commandConfig.command('get')
+        .description('get the crossbind system configuration')
+        .argument('<key>', 'configuration key to read')
+        .action((key) => getSystemConfig(key));
+
+    commandConfig.command('set')
+        .description('set the crossbind system configuration')
+        .argument('<key>', 'configuration key to set')
+        .argument('<value>', 'value to assign')
+        .action((key, value) => setSystemConfig(key, value));
+
+    commandConfig.command('delete')
+        .description('delete the crossbind system configuration')
+        .argument('<key>', 'configuration key to remove')
+        .action((key) => deleteSystemConfig(key));
+
+    commandConfig.command('list')
+        .description('list the crossbind configurations')
+        .addOption(new Option('-t, --type <type>', 'config type').default('system').choices(['all', 'system', 'project']))
+        .action((options) => listSystemConfig(options.type));
+
+    commandConfig.command('keys')
+        .description('list all available system configuration keys for crossbind')
+        .action(() => listSystemKeys());
+}
+
+function listSystemKeys() {
+    console.info('Available configurations:');
+    console.table(systemKeys);
+}
+
+async function getSystemConfig(systemKey) {
+    if (!systemKey) {
+        await listSystemConfig('system');
+        return;
+    }
+    const system = loadSystemConfig();
+    const key = systemKey.toUpperCase();
+    const output = {};
+    if (key in system) {
+        output[key] = { value: system[key] || 'undefined', default: systemKeys[key].default };
+    }
+    Object.keys(systemKeys).filter((k) => k.includes(key)).forEach((k) => {
+        if (k in system) {
+            output[k] = { value: system[k] || 'undefined', default: systemKeys[k].default };
+        }
+    });
+
+    console.table(output);
+}
+
+function setSystemConfig(key, value) {
+    if (!systemKeys[key]) {
+        throw new Error(`Configuration ${key} is not available. Please choose from the following available configurations: ${Object.keys(systemKeys).join(', ')}`);
+    }
+    if (systemKeys[key].options && !systemKeys[key].options.includes(value)) {
+        throw new Error(`Value ${value} is not available. Please choose from the following available values: ${systemKeys[key].options.join(', ')}`);
+    }
+
+    writeJson(systemConfigPath(), { ...loadSystemConfig(), [key]: value });
+}
+
+function deleteSystemConfig(key) {
+    if (!systemKeys[key]) {
+        throw new Error(`Configuration ${key} is not available. Please choose from the following available configurations: ${Object.keys(systemKeys).join(', ')}`);
+    }
+    const { [key]: deleted, ...system } = loadSystemConfig();
+    writeJson(systemConfigPath(), system);
+}
+
+async function listSystemConfig(type) {
+    // Importing the state loads the project, which fails outside one, so only a listing of the project does.
+    const { system: systemConfig, ...projectConfig } = type === 'system'
+        ? { system: loadSystemConfig() }
+        : (await import('./state/index.js')).default.config;
+    if (type === 'all' || type === 'system') {
+        console.log('System Configuration');
+        console.table(systemConfig);
+    }
+
+    if (type === 'all') {
+        console.log('');
+    }
+
+    if (type === 'all' || type === 'project') {
+        console.log(`Project Configuration (paths relative to ${projectConfig.paths.base})`);
+        console.table(flattenConfigForTable(projectConfig, { base: projectConfig.paths.base }));
+    }
+}

package/src/runner/blobs.js

@@ -0,0 +1,135 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { pipeline } from 'node:stream/promises';
+import { sha256Of } from './files.js';
+
+const SHA256 = /^[0-9a-f]{64}$/;
+const UPLOAD_ID = /^[\w-]{8,64}$/;
+// A part nobody added to for an hour belongs to an upload that stopped; the disk may be memory (Cloud Run).
+const STALE_PART_MS = 60 * 60 * 1000;
+
+export const isHash = (value) => typeof value === 'string' && SHA256.test(value);
+export const isUploadId = (value) => typeof value === 'string' && UPLOAD_ID.test(value);
+
+// Content-addressed: a blob is stored under its sha256, and an upload whose bytes hash to something else is dropped.
+export function createBlobStore(dir) {
+    fs.mkdirSync(dir, { recursive: true });
+    const root = `${path.resolve(dir)}${path.sep}`;
+
+    const pathOf = (hash) => {
+        if (!isHash(hash)) throw new Error(`crossbind runner: "${hash}" is not a sha256 digest`);
+        return path.join(dir, hash);
+    };
+
+    const has = (hash) => isHash(hash) && fs.existsSync(path.join(dir, hash));
+
+    async function putStream(hash, stream) {
+        const temp = path.join(dir, `.${hash}.${process.pid}.${crypto.randomUUID()}`);
+        const digest = crypto.createHash('sha256');
+        const output = fs.createWriteStream(temp);
+        try {
+            await pipeline(stream, async function* hashing(source) {
+                for await (const chunk of source) {
+                    digest.update(chunk);
+                    yield chunk;
+                }
+            }, output);
+        } catch (error) {
+            // pipeline settles before a file that is still opening closes; removed earlier, it would come back.
+            if (!output.closed) await new Promise((resolve) => { output.once('close', resolve); });
+            fs.rmSync(temp, { force: true });
+            throw error;
+        }
+        if (digest.digest('hex') !== hash) {
+            fs.rmSync(temp, { force: true });
+            return false;
+        }
+        fs.renameSync(temp, pathOf(hash));
+        return true;
+    }
+
+    function dropStaleParts() {
+        const now = Date.now();
+        for (const name of fs.readdirSync(dir).filter((file) => file.endsWith('.part'))) {
+            const stat = fs.statSync(path.join(dir, name), { throwIfNoEntry: false });
+            if (stat && now - stat.mtimeMs > STALE_PART_MS) fs.rmSync(path.join(dir, name), { force: true });
+        }
+    }
+
+    // A request names the part file, so its path is checked to stay in the store.
+    const partOf = (hash, upload) => {
+        const file = path.resolve(root, `.${hash}.${upload}.part`);
+        if (!file.startsWith(root)) throw new Error(`crossbind runner: "${hash}" and "${upload}" name no part in the store`);
+        return file;
+    };
+
+    // A blob too large for one request arrives in parts, in order and under the id of its upload; the last part
+    // checks the joined file against the hash. A part that breaks off leaves the ones before it, so the client sends
+    // that part again, and a part of a blob already joined answers as stored, since the last answer may have been lost.
+    async function writePart(hash, upload, { start, end, total }, stream) {
+        if (!isUploadId(upload)) throw new Error(`crossbind runner: "${upload}" is not an upload id`);
+        if (has(hash)) {
+            for await (const chunk of stream) void chunk;
+            return { status: 'stored' };
+        }
+        if (start === 0) dropStaleParts();
+        const partial = partOf(hash, upload);
+        const received = fs.existsSync(partial) ? fs.statSync(partial).size : 0;
+        if (start !== received) return { status: 'gap', received };
+        const restore = () => {
+            if (start === 0) fs.rmSync(partial, { force: true });
+            else if (fs.existsSync(partial)) fs.truncateSync(partial, start);
+        };
+        let length = 0;
+        const output = fs.createWriteStream(partial, { flags: 'a' });
+        try {
+            await pipeline(stream, async function* counting(source) {
+                for await (const chunk of source) {
+                    length += chunk.length;
+                    yield chunk;
+                }
+            }, output);
+        } catch (error) {
+            if (!output.closed) await new Promise((resolve) => { output.once('close', resolve); });
+            restore();
+            throw error;
+        }
+        if (length !== end - start + 1) {
+            restore();
+            return { status: 'short' };
+        }
+        if (end + 1 < total) return { status: 'partial', received: end + 1 };
+        const digest = crypto.createHash('sha256');
+        for await (const chunk of fs.createReadStream(partial)) digest.update(chunk);
+        const joined = digest.digest('hex');
+        if (joined !== hash) {
+            fs.rmSync(partial, { force: true });
+            return { status: 'mismatch' };
+        }
+        fs.renameSync(partial, pathOf(joined));
+        return { status: 'stored' };
+    }
+
+    // The parts of one upload go one at a time: a part whose connection broke off can still be writing when the client
+    // sends it again.
+    const writing = new Map();
+    function putPart(hash, upload, range, stream) {
+        const key = `${hash}.${upload}`;
+        const current = (writing.get(key) ?? Promise.resolve()).catch(() => {}).then(() => writePart(hash, upload, range, stream));
+        writing.set(key, current);
+        const forget = () => {
+            if (writing.get(key) === current) writing.delete(key);
+        };
+        current.then(forget, forget);
+        return current;
+    }
+
+    function putBuffer(data) {
+        const hash = sha256Of(data);
+        if (!has(hash)) fs.writeFileSync(pathOf(hash), data);
+        return hash;
+    }
+
+    return { pathOf, has, putStream, putPart, putBuffer };
+}