crossbind 2.0.0-beta.62 → 2.0.0-beta.64

package/src/actions/buildWasm.js

@@ -1,9 +1,10 @@
 import fs from 'node:fs';
+import { writeNativeSourceStamp } from '../utils/nativeSourceStamp.js';
 import { fileURLToPath } from 'node:url';
-import replace from 'replace';
 import run from './run.js';
 import getLinkInputs from './getLinkInputs.js';
 import getData from './getData.js';
+import getEnv from './getEnv.js';
 import buildJs from './buildJs.js';
 import triggerExtensions from './extensions.js';
 import state from '../state/index.js';
@@ -12,6 +13,13 @@ import logger from '../utils/logger.js';
 import { getContentHash, getFilesFingerprint } from '../utils/hash.js';
 import { buildLinkLibArgs } from '../utils/linkLayout.js';
 import { guardBigIntArguments, guardEmbindArguments } from '../utils/embindArgumentGuards.js';
+import { separateCallArguments } from '../utils/embindCallArguments.js';
+import { queueJspiCalls } from '../utils/embindJspiQueue.js';
+import { releaseOwnedHandles } from '../utils/embindObjectLifetime.js';
+import { spawnPthreadsFromMainScript } from '../utils/pthreadMainScript.js';
+
+const GLUE_REWRITES = ['../utils/embindArgumentGuards.js', '../utils/embindCallArguments.js', '../utils/embindJspiQueue.js', '../utils/embindObjectLifetime.js', '../utils/pthreadMainScript.js']
+    .map((file) => fileURLToPath(new URL(file, import.meta.url)));
 
 // embind's bigint converter turns any Number into a BigInt, so 2^53+1 silently becomes 2^53.
 // A 64-bit parameter takes a BigInt or a Number that is a safe integer instead; the jsi adapter
@@ -36,15 +44,52 @@ function guardArgumentConversions(target) {
     }
 }
 
+// Overlapping calls of one embind function free each other's arguments (utils/embindCallArguments.js);
+// each runtimeEnv links its own glue, so each one calls this after its link.
+function separateEmbindCalls(target) {
+    const gluePath = `${state.config.paths.build}/${target.rawJsName}`;
+    const { text, missed } = separateCallArguments(fs.readFileSync(gluePath, 'utf8'));
+    fs.writeFileSync(gluePath, text);
+    if (missed) {
+        logger.error('embind per-call argument rewrite missed (emscripten glue format changed?): overlapping calls of one function free each other\'s arguments');
+    }
+}
+
+// Suspended _JSPI calls that resume out of order overwrite each other's C stack (utils/embindJspiQueue.js);
+// each runtimeEnv links its own glue, so each one calls this after its link.
+function queueJspiEmbindCalls(target) {
+    const gluePath = `${state.config.paths.build}/${target.rawJsName}`;
+    const { text, missed } = queueJspiCalls(fs.readFileSync(gluePath, 'utf8'));
+    fs.writeFileSync(gluePath, text);
+    if (missed) {
+        logger.error('embind _JSPI queue rewrite missed (emscripten glue format changed?): suspended _JSPI calls can overwrite each other\'s C stack');
+    }
+}
+
+// Objects JavaScript owns outside a smart pointer were freed only by .delete() (utils/embindObjectLifetime.js);
+// each runtimeEnv links its own glue, so each one calls this after its link.
+function releaseOwnedObjects(target) {
+    const gluePath = `${state.config.paths.build}/${target.rawJsName}`;
+    const { text, missed } = releaseOwnedHandles(fs.readFileSync(gluePath, 'utf8'));
+    fs.writeFileSync(gluePath, text);
+    if (missed) {
+        logger.error('embind object lifetime rewrite missed (emscripten glue format changed?): returned values and constructed vectors are freed only by .delete()');
+    }
+}
+
+// A direct-mode mt init in a browser spawns its pthreads from "/undefined" (utils/pthreadMainScript.js).
+function pointPthreadSpawn(target) {
+    const gluePath = `${state.config.paths.build}/${target.rawJsName}`;
+    const { text, missed } = spawnPthreadsFromMainScript(fs.readFileSync(gluePath, 'utf8'));
+    fs.writeFileSync(gluePath, text);
+    if (missed) {
+        logger.error('pthread spawn rewrite missed (emscripten glue format changed?): mt builds may fetch /undefined workers');
+    }
+}
+
 export default async function buildWasm(target, options = {}) {
     const isProd = target.buildType === 'release';
 
-    // Caller can opt out of the final emcc link entirely (e.g. when the
-    // package is consumed only as a static library by downstream builds).
-    if (state.config.export.bundle === false) {
-        logger.info(`[${target.path}] wasm+js skipped (export.bundle = false)`);
-        return false;
-    }
     // Cargo packages only stage their staticlib; the consuming app links and bundles it.
     if (state.config.export.type === 'cargo') {
         logger.info(`[${target.path}] wasm+js skipped (cargo package - staticlib only)`);
@@ -131,7 +176,8 @@ export default async function buildWasm(target, options = {}) {
         // The emcc arg tail is hardcoded per environment in this file, invisible to the
         // emccFlags entry below - hashing the builder itself makes any inline-arg edit
         // (a new -s flag, an EXPORTED_RUNTIME_METHODS change) a guaranteed cache miss.
-        builder: getFilesFingerprint([fileURLToPath(import.meta.url)]),
+        // The glue rewrites applied after the link count as the builder too.
+        builder: getFilesFingerprint([fileURLToPath(import.meta.url), ...GLUE_REWRITES]),
         wholeArchiveAll,
         wholeArchiveNames: [...wholeArchiveNames].sort(),
         emccFlags,
@@ -140,6 +186,8 @@ export default async function buildWasm(target, options = {}) {
             return { lib, size: stat ? stat.size : null, mtimeMs: stat ? stat.mtimeMs : null };
         }),
         data: getData('data', target),
+        // The JS the link step writes carries the env.
+        env: getEnv(target),
         runtime: getFilesFingerprint(runtimeAssets),
         rustSources: getFilesFingerprint(rustSources),
     }));
@@ -188,27 +236,7 @@ export default async function buildWasm(target, options = {}) {
         const t1 = performance.now();
         logger.doneStep(target, 'wasm');
         logger.startStep(target, 'js');
-        // The pthread bootstrap spawns workers from _scriptName, captured from
-        // document.currentScript at LOAD time - undefined when the bridge is
-        // loaded as a module script, so workers would fetch "/undefined" and
-        // direct-mode mt init hangs. Inject Module.crossbindMainScript (set by the
-        // browser adapter from paths.worker/paths.js; a crossbind-specific key, as
-        // emscripten's own mainScriptUrlOrBlob trips emsdk 6 debug assertions
-        // on st targets) at SPAWN time instead. The previous spaced
-        // 'var _scriptName = ' rewrite no longer matched the minified glue and
-        // silently no-opped; the guard below turns any future format drift
-        // into a visible error instead.
-        replace({
-            regex: 'pthreadMainJs=_scriptName',
-            replacement: 'pthreadMainJs=Module["crossbindMainScript"]||_scriptName',
-            paths: [`${state.config.paths.build}/${target.rawJsName}`],
-            recursive: false,
-            silent: true,
-        });
-        const glue = fs.readFileSync(`${state.config.paths.build}/${target.rawJsName}`, 'utf8');
-        if (glue.includes('pthreadMainJs') && !glue.includes('Module["crossbindMainScript"]||_scriptName')) {
-            logger.error('pthread spawn rewrite missed (emscripten glue format changed?): mt builds may fetch /undefined workers');
-        }
+        pointPthreadSpawn(target);
         /* replace({
             regex: 'val === 10',
             replacement: 'false',
@@ -218,6 +246,9 @@ export default async function buildWasm(target, options = {}) {
         }); */
         guardBigIntConversions(target);
         guardArgumentConversions(target);
+        separateEmbindCalls(target);
+        queueJspiEmbindCalls(target);
+        releaseOwnedObjects(target);
         await buildJs(target);
         // fs.rmSync(`${state.config.paths.build}/${state.config.general.name}.js`);
         // fs.copyFileSync(`${state.config.paths.build}/${state.config.general.name}.browser.js`, `${state.config.paths.build}/${state.config.general.name}.js`);
@@ -242,12 +273,13 @@ export default async function buildWasm(target, options = {}) {
             // it any linked Rust package archive fails with undefined crossbind_* symbols.
             ...linkLibs, ...rustSources,
             '-s', 'WASM=1', '-s', 'MODULARIZE=1', '-s', 'DYNAMIC_EXECUTION=0',
-            '-s', 'RESERVED_FUNCTION_POINTERS=200', // '-s', 'FORCE_FILESYSTEM=1',
+            '-s', 'RESERVED_FUNCTION_POINTERS=200',
             '-s', 'ALLOW_MEMORY_GROWTH=1',
             // See the GROWABLE_ARRAYBUFFERS note in the browser block (Firefox/WebKit TextDecoder).
             '-s', 'GROWABLE_ARRAYBUFFERS=0',
+            '-s', 'FORCE_FILESYSTEM=1',
             '-s', 'ENVIRONMENT=web',
-            '-s', 'EXPORTED_RUNTIME_METHODS=["ENV", "getExceptionMessage"]',
+            '-s', 'EXPORTED_RUNTIME_METHODS=["FS", "ENV", "getExceptionMessage"]',
             '-fwasm-exceptions',
             '-sEXPORT_EXCEPTION_HANDLING_HELPERS',
             '-o', `${state.config.paths.build}/${target.rawJsName}`,
@@ -258,6 +290,9 @@ export default async function buildWasm(target, options = {}) {
         logger.startStep(target, 'js');
         guardBigIntConversions(target);
         guardArgumentConversions(target);
+        separateEmbindCalls(target);
+        queueJspiEmbindCalls(target);
+        releaseOwnedObjects(target);
         await buildJs(target);
         logger.doneStep(target, 'js');
     }
@@ -275,6 +310,8 @@ export default async function buildWasm(target, options = {}) {
             // '-s', 'FETCH', '-sJSPI', '-pthread', '-sPTHREAD_POOL_SIZE=5',
             '-s', 'FORCE_FILESYSTEM=1',
             ...linkLibs, ...rustSources, `${state.config.paths.cli}/assets/cpp-runtime/node.cpp`,
+            // node.cpp answers fsync and fdatasync for host files.
+            '-Wl,--wrap=__wasi_fd_sync', '-Wl,--wrap=__syscall_fdatasync',
             '-s', 'WASM=1', '-s', 'MODULARIZE=1', '-s', 'DYNAMIC_EXECUTION=0',
             '-s', 'RESERVED_FUNCTION_POINTERS=200', // '-s', 'DISABLE_EXCEPTION_CATCHING=0', '-s', 'FORCE_FILESYSTEM=1',
             '-s', 'ALLOW_MEMORY_GROWTH=1',
@@ -292,6 +329,9 @@ export default async function buildWasm(target, options = {}) {
         logger.startStep(target, 'js');
         guardBigIntConversions(target);
         guardArgumentConversions(target);
+        separateEmbindCalls(target);
+        queueJspiEmbindCalls(target);
+        releaseOwnedObjects(target);
         await buildJs(target);
         if (emccFlags.includes('FETCH')) {
             fs.appendFileSync(`${state.config.paths.build}/${target.jsName}`, 'var XMLHttpRequest = require(\'xhr2\');\n');
@@ -315,5 +355,6 @@ export default async function buildWasm(target, options = {}) {
     }
 
     fs.writeFileSync(linkFingerprintFile, linkFingerprint);
+    writeNativeSourceStamp(`${state.config.paths.build}/${target.jsName}`, state.config.paths.native);
     return true;
 }

package/src/actions/cloudCommands.js

@@ -0,0 +1,133 @@
+import {
+    CREDENTIALS_FILE, cloudCredentials, readCredentials, writeCredentials,
+} from '../utils/cloudAccount.js';
+
+// `crossbind login`, `logout` and `usage`, against tooling/cloud/worker/account-api.js.
+
+const GITHUB_ORIGIN = 'https://github.com';
+const TOKEN_PATTERN = /^cbt_[A-Za-z0-9_-]{43}$/;
+const MAX_NAME_LENGTH = 64;
+// GitHub's defaults for a device code; it asks for no faster polling than this.
+const MIN_INTERVAL_SECONDS = 5;
+const DEFAULT_EXPIRES_SECONDS = 900;
+
+const wait = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); });
+// What the cloud sends ends up on the terminal: no escape sequences.
+const printable = (value) => String(value ?? '').replace(/\p{Cc}/gu, '');
+const paceOf = (seconds, fallback) => Math.max(MIN_INTERVAL_SECONDS, Number.isFinite(seconds) ? seconds : fallback);
+
+async function call(fetcher, url, route, { method = 'GET', token, body } = {}) {
+    let response;
+    try {
+        // A redirect would carry the token to an address nobody configured.
+        response = await fetcher(`${url}${route}`, {
+            method,
+            headers: { ...(token ? { authorization: `Bearer ${token}` } : {}), ...(body ? { 'content-type': 'application/json' } : {}) },
+            body: body && JSON.stringify(body),
+            redirect: 'error',
+        });
+    } catch (error) {
+        throw new Error(`crossbind: crossbind cloud at ${url} is unreachable (${error.cause?.code ?? error.message}).`, { cause: error });
+    }
+    const answer = await response.json().catch(() => null);
+    return { status: response.status, ok: response.ok, body: answer !== null && typeof answer === 'object' ? answer : {} };
+}
+
+const refused = ({ status, body }) => new Error(`crossbind: ${printable(body.error ?? `crossbind cloud answered ${status}.`)}`);
+
+// { revoked: true }, or { revoked: false, reason }.
+async function revoke(fetcher, url, token, all) {
+    try {
+        const answer = await call(fetcher, url, all ? '/v1/tokens' : '/v1/token', { method: 'DELETE', token });
+        // 401: the cloud no longer knows this token, so there is nothing left to revoke, unless every token was asked for.
+        return answer.ok || (!all && answer.status === 401) ? { revoked: true } : { revoked: false, reason: refused(answer).message };
+    } catch (error) {
+        return { revoked: false, reason: error.message };
+    }
+}
+
+// GitHub's device flow: the user approves a code on github.com while this polls crossbind cloud, which asks GitHub.
+export async function login({
+    url, file = CREDENTIALS_FILE, fetcher = fetch, sleep = wait, now = Date.now, print = console.log,
+}) {
+    const device = await call(fetcher, url, '/v1/login/device', { method: 'POST' });
+    if (!device.ok) throw refused(device);
+    const { deviceCode, userCode, verificationUri } = device.body;
+    if (URL.parse(verificationUri)?.origin !== GITHUB_ORIGIN || typeof userCode !== 'string') {
+        throw new Error('crossbind: crossbind cloud sent a sign-in page that is not on github.com.');
+    }
+    print(`crossbind: open ${printable(verificationUri)} and enter the code ${printable(userCode)} to sign in with GitHub.`);
+    let pace = paceOf(device.body.interval, MIN_INTERVAL_SECONDS);
+    const deadline = now() + (device.body.expiresIn ?? DEFAULT_EXPIRES_SECONDS) * 1000;
+    while (now() < deadline) {
+        await sleep(pace * 1000);
+        const answer = await call(fetcher, url, '/v1/login/token', { method: 'POST', body: { deviceCode } });
+        if (answer.status === 202) {
+            pace = paceOf(answer.body.interval, pace);
+        } else {
+            if (!answer.ok) throw refused(answer);
+            const { token } = answer.body;
+            if (!TOKEN_PATTERN.test(token ?? '')) throw new Error('crossbind: crossbind cloud sent no usable token.');
+            const name = printable(answer.body.login).slice(0, MAX_NAME_LENGTH);
+            const previous = cloudCredentials(url, file);
+            writeCredentials({ ...readCredentials(file), [url]: { token, login: name } }, file);
+            if (previous && previous.token !== token) {
+                const outcome = await revoke(fetcher, url, previous.token, false);
+                if (!outcome.revoked) print(`crossbind: the token this sign-in replaces still works (${outcome.reason.replace(/^crossbind: /, '')}); crossbind logout --all revokes it.`);
+            }
+            return { login: name };
+        }
+    }
+    throw new Error('crossbind: the code expired before it was entered; run crossbind login again.');
+}
+
+// Forgets the token here whatever the cloud answers; `revoked` says whether the cloud dropped it too.
+export async function logout({
+    url, all = false, file = CREDENTIALS_FILE, fetcher = fetch,
+}) {
+    const { [url]: entry, ...rest } = readCredentials(file);
+    if (!entry) return null;
+    const outcome = await revoke(fetcher, url, entry.token, all);
+    writeCredentials(rest, file);
+    return { login: entry.login, ...outcome };
+}
+
+// The account, with the sign-in of every machine; the sign-in here is forgotten only once the cloud confirms.
+export async function deleteAccount({ url, file = CREDENTIALS_FILE, fetcher = fetch }) {
+    const { [url]: entry, ...rest } = readCredentials(file);
+    if (!entry) throw new Error(`crossbind: not signed in to ${url}; run crossbind login first.`);
+    const answer = await call(fetcher, url, '/v1/account', { method: 'DELETE', token: entry.token });
+    if (answer.status === 401) throw new Error('crossbind: this sign-in is no longer valid; run crossbind login, then delete the account.');
+    if (!answer.ok) throw refused(answer);
+    writeCredentials(rest, file);
+    return { login: entry.login };
+}
+
+export async function fetchUsage({ url, file = CREDENTIALS_FILE, fetcher = fetch }) {
+    const entry = cloudCredentials(url, file);
+    if (!entry) throw new Error(`crossbind: not signed in to ${url}; run crossbind login.`);
+    const answer = await call(fetcher, url, '/v1/usage', { token: entry.token });
+    if (answer.status === 401) throw new Error('crossbind: this sign-in is no longer valid; run crossbind login.');
+    if (!answer.ok) throw refused(answer);
+    return answer.body;
+}
+
+function formatSeconds(seconds) {
+    const whole = Math.max(0, Math.round(seconds));
+    const minutes = Math.floor(whole / 60);
+    const rest = whole % 60;
+    if (minutes === 0) return `${rest}s`;
+    return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`;
+}
+
+export function usageLines({ login: name, builds, playground }) {
+    const images = Object.entries(builds.byImage).sort(([, a], [, b]) => b - a);
+    const column = Math.max(...images.map(([image]) => image.length)) + 3;
+    const left = builds.limitSeconds - builds.usedSeconds;
+    return [
+        `Signed in as ${printable(name)}.`,
+        `Cloud builds in ${printable(builds.month)}: ${formatSeconds(builds.usedSeconds)} of ${formatSeconds(builds.limitSeconds)} used, ${formatSeconds(left)} left.`,
+        ...images.map(([image, seconds]) => `  ${printable(image).padEnd(column)}${formatSeconds(seconds)}`),
+        `Playground: ${playground.remaining} of ${playground.limit} compiles left today.`,
+    ];
+}

package/src/actions/createImportedBridges.js

@@ -0,0 +1,71 @@
+import fs from 'node:fs';
+import upath from 'upath';
+import state from '../state/index.js';
+import createBridgeFile from './createInterface.js';
+import getDependFilePath from '../integration/getDependFilePath.js';
+import resolveNativeImport from '../integration/resolveNativeImport.js';
+import { getRustSymbols } from '../integration/getCrossbindScript.js';
+import { findNativeImportsIn, nativeSpecifierTest } from '../utils/nativeImports.js';
+
+const SCHEME = /^(cargo|conan):/;
+const isBare = (specifier) => !specifier.startsWith('.') && !upath.isAbsolute(specifier);
+
+function realPath(file) {
+    try {
+        return upath.normalize(fs.realpathSync(file));
+    } catch (e) {
+        return upath.resolve(file);
+    }
+}
+
+// The first target whose packages ship the file. A cargo: or conan: import that none resolves fails the build with
+// what to declare or install, as it does in a bundler.
+function resolveImport(specifier, importer, targets) {
+    let failure = null;
+    for (const target of targets) {
+        try {
+            const file = SCHEME.test(specifier) ? getDependFilePath(specifier, target) : resolveNativeImport(specifier, importer, target);
+            if (file && fs.existsSync(file)) return { target, file };
+        } catch (error) {
+            failure ??= error;
+        }
+    }
+    if (failure) throw failure;
+    return null;
+}
+
+// Whether a binary binds what the app's own sources import. A Node binary does: its module hooks resolve those
+// imports. A browser binary only with --bind-imports: without a bundler a page cannot resolve a header import
+// unless it maps the specifiers itself, as the playground's sandbox does. Nothing on an edge runtime resolves one.
+export function bindsAppImports(target, { bindImports = false } = {}) {
+    return target.runtimeEnv === 'node' || (bindImports && target.runtimeEnv === 'browser');
+}
+
+// The native files the app's own sources import, by real path, each under the first target whose packages ship it
+// and with the bare specifiers that name it, which Node cannot resolve on its own.
+export function findAppNativeImports(targets) {
+    const projectDir = state.config.paths.project;
+    const imports = new Map();
+    if (!projectDir || !fs.existsSync(projectDir)) return imports;
+    for (const { importer, specifier } of findNativeImportsIn(projectDir, nativeSpecifierTest(state.config.ext))) {
+        const resolved = resolveImport(specifier, importer, targets);
+        if (!resolved) {
+            console.warn(`crossbind: ${upath.relative(projectDir, importer)} imports ${specifier}, which no dependency ships for ${targets.map((t) => t.path).join(', ')}.`);
+            continue;
+        }
+        const file = realPath(resolved.file);
+        const found = imports.get(file) ?? { file, target: resolved.target, specifiers: [] };
+        const specifiers = isBare(specifier) && !found.specifiers.includes(specifier) ? [...found.specifiers, specifier] : found.specifiers;
+        imports.set(file, { ...found, specifiers });
+    }
+    return imports;
+}
+
+// The imported files no bridge of the build covers yet, bound the way a bundler plugin binds what it transforms. A Rust
+// import has no bridge file: its crate links through buildAppRustCrates or its package, so it brings its names.
+export function createImportedBridges(bound, imports) {
+    const boundFiles = new Set(bound.map(({ file }) => realPath(file)));
+    return [...imports.values()].filter(({ file }) => !boundFiles.has(file)).map(({ file, target }) => (file.endsWith('.rs')
+        ? { file, names: getRustSymbols(file) }
+        : { file, bridge: createBridgeFile(file, target) }));
+}