@nimbus-sh/worker 0.12.0 → 0.13.0

package/scripts/bundle-oxc-wasm.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+/**
+ * bundle-oxc-wasm.mjs — stage Nimbus's Oxc transform for the transform facet.
+ *
+ * scripts/oxc-wasm/build.mjs builds the crate in scripts/oxc-wasm into
+ *
+ *   nimbus-oxc.wasm, provenance.json
+ *
+ * This script does NOT rebuild (that needs the pinned Rust toolchain), and
+ * refuses a wasm whose provenance records other inputs than the tree's
+ * (scripts/oxc-wasm/inputs.mjs: the crate, its lockfile and toolchain pin,
+ * the recipe and its compiler wrapper), so a Rust change cannot ship the
+ * previous build. It stages that pair into the static-assets layer under a name carrying a
+ * prefix of the wasm's digest, so no cache layer can serve another build's
+ * bytes under it, and pins the digest and size in
+ * src/oxc-wasm-artifact.generated.ts, which runtime/oxc-wasm-bytes.ts verifies
+ * on every fetch:
+ *
+ *   public/_assets/oxc/nimbus-oxc-<build id>.wasm
+ *   public/_assets/oxc/nimbus-oxc-<build id>.provenance.json
+ *
+ * Source: NIMBUS_OXC_WASM_ARTIFACTS (a build.mjs --out directory), else the
+ * committed public/_assets/oxc/ (the artifact is tracked, so a checkout
+ * without the Rust build re-derives the same generated module).
+ *
+ * Run via: node scripts/bundle-oxc-wasm.mjs (wired into `bundle`).
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { changedInputs, oxcWasmInputs } from './oxc-wasm/inputs.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const ASSETS = path.join(ROOT, 'public', '_assets', 'oxc');
+const OUT_TS = path.join(ROOT, 'src', 'oxc-wasm-artifact.generated.ts');
+
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+const fail = (message) => { throw new Error(`[bundle-oxc-wasm] ${message}`); };
+
+/** The wasm and its provenance: a fresh build's, or the committed stage's. */
+async function source() {
+  const built = process.env.NIMBUS_OXC_WASM_ARTIFACTS;
+  if (built) {
+    const dir = path.resolve(built);
+    return { wasm: path.join(dir, 'nimbus-oxc.wasm'), provenance: path.join(dir, 'provenance.json') };
+  }
+  const staged = (await fs.readdir(ASSETS).catch(() => [])).filter((e) => e.endsWith('.wasm'));
+  if (staged.length !== 1) fail(`expected one staged wasm under ${ASSETS}, found ${JSON.stringify(staged)}; set NIMBUS_OXC_WASM_ARTIFACTS`);
+  const wasm = path.join(ASSETS, staged[0]);
+  return { wasm, provenance: wasm.replace(/\.wasm$/, '.provenance.json') };
+}
+
+const src = await source();
+const wasm = await fs.readFile(src.wasm);
+const provenanceText = await fs.readFile(src.provenance, 'utf8');
+const provenance = JSON.parse(provenanceText);
+const digest = sha256(wasm);
+if (provenance.artifact !== 'nimbus-oxc') fail(`${src.provenance} describes ${provenance.artifact}, not nimbus-oxc`);
+if (provenance.outputs?.['nimbus-oxc.wasm']?.sha256 !== digest) {
+  fail(`${src.wasm} sha256 ${digest} does not match its provenance.json (${provenance.outputs?.['nimbus-oxc.wasm']?.sha256})`);
+}
+const stale = changedInputs(provenance.source?.inputs, await oxcWasmInputs());
+if (stale.length) {
+  fail(`${src.wasm} was built from other inputs than this tree's (${stale.join(', ')}); rebuild it with scripts/oxc-wasm/build.mjs and stage that build`);
+}
+const memoryPages = provenance.wasm?.memoryInitialPages;
+if (!Number.isInteger(memoryPages) || memoryPages <= 0) fail(`${src.provenance} records no memoryInitialPages`);
+
+const buildId = digest.slice(0, 16);
+const wasmName = `nimbus-oxc-${buildId}.wasm`;
+const provenanceName = `nimbus-oxc-${buildId}.provenance.json`;
+await fs.mkdir(ASSETS, { recursive: true });
+for (const entry of await fs.readdir(ASSETS)) {
+  if (entry !== wasmName && entry !== provenanceName) await fs.rm(path.join(ASSETS, entry), { recursive: true, force: true });
+}
+if (path.resolve(src.wasm) !== path.join(ASSETS, wasmName)) {
+  await fs.writeFile(path.join(ASSETS, wasmName), wasm);
+  await fs.writeFile(path.join(ASSETS, provenanceName), provenanceText);
+}
+
+await fs.writeFile(OUT_TS, `/**
+ * oxc-wasm-artifact.generated.ts — AUTO-GENERATED by scripts/bundle-oxc-wasm.mjs
+ * DO NOT EDIT.
+ *
+ * Nimbus's Oxc transform (scripts/oxc-wasm, built by its build.mjs; inputs in
+ * the staged provenance.json), staged under public/_assets/oxc/. The transform
+ * facet's builder (runtime/oxc-wasm-bytes.ts) fetches it and verifies it
+ * against OXC_WASM_SHA256 before the loader compiles it. The module imports
+ * nothing; its memory starts at OXC_WASM_MEMORY_PAGES pages (stack and static
+ * data) and grows only while a transform runs.
+ *
+ * oxc ${provenance.oxc}, ${provenance.toolchain.split('\n')[0]}, ${(wasm.length / 1048576).toFixed(2)} MiB
+ */
+
+export const OXC_VERSION: string = ${JSON.stringify(provenance.oxc)};
+export const OXC_WASM_ASSET_PATH: string = ${JSON.stringify(`/_assets/oxc/${wasmName}`)};
+export const OXC_WASM_BUILD_ID: string = ${JSON.stringify(buildId)};
+export const OXC_WASM_SHA256: string = ${JSON.stringify(digest)};
+export const OXC_WASM_BYTES: number = ${wasm.length};
+export const OXC_WASM_MEMORY_PAGES: number = ${memoryPages};
+`);
+console.log(`[bundle-oxc-wasm] staged /_assets/oxc/${wasmName} (${wasm.length} bytes, sha256 ${digest})`);

package/scripts/interpreter-bundle.mjs

@@ -0,0 +1,105 @@
+/**
+ * interpreter-bundle.mjs — the runtime-code interpreter (@nimbus-sh/core
+ * src/interpreter) as a node launch's map carries it: three CommonJS modules.
+ *
+ *   - The primordials (primordials.ts): the built-ins the interpreter calls,
+ *     captured when the launch starts, before any program code runs. The
+ *     launch's runtime loads it at its start; the interpreter requires it.
+ *   - The interpreter, bundled with acorn into one module. It loads late,
+ *     after a program may have replaced built-ins, so its code must call
+ *     none while it loads or parses: acorn is bundled rewritten to reach the
+ *     realm only through captures (acorn-primordials.mjs, parser-realm.ts),
+ *     and esbuild's CommonJS output would reach it too (its interop
+ *     helpers capture Object.defineProperty and walk export lists with
+ *     for-of), so it is bundled as an ES module, which needs no helpers, and
+ *     its one import and its export statement are then rewritten, by their
+ *     syntax tree, to a require and a `module.exports` assignment. The
+ *     bundle states strict mode, which ES modules had implicitly; the
+ *     interpreter's closures depend on it (an assignment to a read-only
+ *     property throws, `this` is not coerced).
+ *   - The host module it runs on (HOST_OPS_SOURCE), which is sloppy at its
+ *     top level so that interpreted sloppy functions are native sloppy
+ *     functions; it cannot be part of a strict bundle.
+ *
+ * esbuild compiles core's TypeScript itself, so all three are built from core
+ * src: bundle-node-shims.mjs stages them, and the tests build them the same
+ * way to test the source they sit beside.
+ */
+
+import { parse } from 'acorn';
+import { build } from 'esbuild';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { primordialAcorn } from './acorn-primordials.mjs';
+import { resolvePackageDir } from './resolve-package-dir.mjs';
+
+/** How the interpreter requires its primordials: the module beside it. */
+export const PRIMORDIALS_FILE = 'interpreter-primordials.js';
+
+/**
+ * The interpreter's ES module bundle as a CommonJS module: its import of the
+ * primordials becomes a require, its export statement `module.exports`.
+ */
+function asCommonJs(esm) {
+  const program = parse(esm, { ecmaVersion: 'latest', sourceType: 'module' });
+  const edits = [];
+  for (const node of program.body) {
+    if (node.type === 'ImportDeclaration') {
+      if (node.source.value !== `./${PRIMORDIALS_FILE}` || node.specifiers.length !== 1 || node.specifiers[0].type !== 'ImportNamespaceSpecifier') {
+        throw new Error(`[interpreter-bundle] the interpreter imports ${node.source.value} unexpectedly`);
+      }
+      edits.push({ start: node.start, end: node.end, text: `const ${node.specifiers[0].local.name} = require(${JSON.stringify(`./${PRIMORDIALS_FILE}`)});` });
+    } else if (node.type === 'ExportNamedDeclaration' && !node.declaration && !node.source) {
+      const fields = node.specifiers.map((s) => `${JSON.stringify(s.exported.name)}: ${s.local.name}`);
+      edits.push({ start: node.start, end: node.end, text: `module.exports = { ${fields.join(', ')} };` });
+    } else if (node.type.startsWith('Export')) {
+      throw new Error(`[interpreter-bundle] the interpreter bundle has an unexpected ${node.type}`);
+    }
+  }
+  let text = esm;
+  for (const edit of edits.sort((a, b) => b.start - a.start)) text = text.slice(0, edit.start) + edit.text + text.slice(edit.end);
+  return `"use strict";\n${text}`;
+}
+
+/**
+ * @param {{ start: string }} options  `start`: a directory @nimbus-sh/core resolves from.
+ * @returns {Promise<{ primordials: string, interpreter: string, ops: string, parser: string }>}
+ *   `parser`: acorn's module as the rewrite made it, before bundling.
+ */
+export async function bundleInterpreter({ start }) {
+  const src = join(resolvePackageDir('@nimbus-sh/core', { start }), 'src/interpreter');
+  const common = { bundle: true, platform: 'neutral', target: 'esnext', mainFields: ['module', 'main'], minify: true, legalComments: 'none', write: false, metafile: true };
+  // esbuild states strict mode in the CommonJS it makes of an ES module.
+  const primordials = await build({ ...common, entryPoints: [join(src, 'primordials.ts')], format: 'cjs' });
+  let parser = null;
+  const interpreter = await build({
+    ...common,
+    entryPoints: [join(src, 'index.ts')],
+    format: 'esm',
+    plugins: [{
+      name: 'primordials',
+      setup(b) {
+        b.onResolve({ filter: /^\.\/primordials\.js$/ }, () => ({ path: `./${PRIMORDIALS_FILE}`, external: true }));
+        // acorn, rewritten to reach the realm only through parser-realm.ts.
+        b.onLoad({ filter: /[\\/]acorn[\\/]dist[\\/]acorn\.mjs$/ }, async (args) => {
+          if (parser !== null) throw new Error(`[interpreter-bundle] acorn loaded twice (${args.path})`);
+          parser = primordialAcorn(await readFile(args.path, 'utf8'), join(src, 'parser-realm.ts')).code;
+          return { contents: parser, loader: 'js' };
+        });
+      },
+    }],
+  });
+  const inputs = Object.keys(interpreter.metafile.inputs);
+  const acornInputs = inputs.filter((path) => /(^|[\\/])acorn[\\/]/.test(path));
+  if (parser === null || acornInputs.length !== 1) {
+    throw new Error(`[interpreter-bundle] acorn must be bundled once, from dist/acorn.mjs, rewritten; bundled ${acornInputs.join(', ') || 'none'}`);
+  }
+  // HOST_OPS_SOURCE is built when its module loads: compile and load it to read it.
+  const ops = await build({ entryPoints: [join(src, 'host-ops.ts')], bundle: true, format: 'esm', platform: 'neutral', write: false });
+  const { HOST_OPS_SOURCE } = await import(`data:text/javascript;base64,${Buffer.from(ops.outputFiles[0].text).toString('base64')}`);
+  if (typeof HOST_OPS_SOURCE !== 'string' || HOST_OPS_SOURCE.length === 0) {
+    throw new Error('[interpreter-bundle] core src/interpreter/host-ops.ts has no HOST_OPS_SOURCE');
+  }
+  return { primordials: primordials.outputFiles[0].text, interpreter: asCommonJs(interpreter.outputFiles[0].text), ops: HOST_OPS_SOURCE, parser };
+}

package/scripts/napi-wasm/build.mjs

@@ -5,6 +5,8 @@
  *
  *   bun packages/worker/scripts/napi-wasm/build.mjs --work <dir> --out <dir> [--spec rolldown,satteri]
  *
+ * `--spec none` builds the loader alone (no Rust toolchain needed).
+ *
  * Keep this Bun orchestrator at <=4 GiB. If Cargo needs a separate 16 GiB
  * cgroup, name the bounded runner explicitly; builds remain serial:
  *   NIMBUS_CARGO_RUNNER=/mnt/scratch/nimbus/run-bounded \
@@ -60,7 +62,8 @@ function arg(name, required = true) {
 }
 const WORK = path.resolve(arg('--work'));
 const OUT = path.resolve(arg('--out'));
-const SELECTED = (arg('--spec', false) ?? Object.keys(SPECS).join(',')).split(',');
+const specArg = arg('--spec', false) ?? Object.keys(SPECS).join(',');
+const SELECTED = specArg === 'none' ? [] : specArg.split(',');
 for (const name of SELECTED) if (!SPECS[name]) throw new Error(`napi-wasm: no spec named ${name}`);
 
 const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');

package/scripts/napi-wasm/loader/napi-wasm-loader.ts

@@ -63,8 +63,22 @@ export interface NapiWasmBindingHost {
   trampoline: WebAssembly.Module;
   /** Wasm pages the binding's imported memory starts at (its declared minimum). */
   memoryPages: number;
+  /**
+   * The binding's linear memory, when the host makes it (to watch what the
+   * binding grows to): `memoryPages` initial pages, maximum 65536, unshared.
+   * Absent: the loader makes it.
+   */
+  memory?: WebAssembly.Memory;
   /** The binding's name, for diagnostics. */
   name: string;
+  /**
+   * Called once if the binding dies: a trap (a Rust panic aborts), or the
+   * host's stack overflowing inside it (a RangeError). The instance is
+   * unusable from then on and no promise it holds will settle, so the host
+   * settles its own callers and drops the binding. Absent: the error is
+   * thrown as an uncaught one.
+   */
+  onFatal?(error: unknown): void;
 }
 
 /** What the loader calls on the binding instance besides napi. */
@@ -500,7 +514,7 @@ export function createNapiWasmBinding(host: NapiWasmBindingHost): Record<string,
   // stack may suspend, so only such a binding takes the JSPI trampoline.
   const pumped = WebAssembly.Module.exports(host.binding).some((e) => e.name === 'nimbus_napi_pump');
   const jspi = pumped && typeof WebAssembly.Suspending === 'function' && typeof WebAssembly.promising === 'function';
-  const memory = new WebAssembly.Memory({ initial: host.memoryPages, maximum: 65536 });
+  const memory = host.memory ?? new WebAssembly.Memory({ initial: host.memoryPages, maximum: 65536 });
   const getMemory = () => memory;
   const fds = new Map<number, FilesystemFd>([
     [0, { kind: 'stdin' }],
@@ -562,9 +576,15 @@ export function createNapiWasmBinding(host: NapiWasmBindingHost): Record<string,
   let fatal: unknown = null;
 
   const die = (error: unknown) => {
+    if (fatal !== null) return;
     fatal = error;
-    // A trap (a Rust panic aborts) leaves the instance unusable. Surface it
-    // as an uncaught error rather than leave every pending promise hanging.
+    while (waiting > 0) context.decreaseWaitingRequestCounter();
+    // A trap or a stack overflow leaves the instance unusable: tell the host,
+    // or surface it as an uncaught error.
+    if (host.onFatal) {
+      host.onFatal(error);
+      return;
+    }
     queueMicrotask(() => {
       throw error;
     });
@@ -636,6 +656,21 @@ export function createNapiWasmBinding(host: NapiWasmBindingHost): Record<string,
   };
 
   const context = createContext();
+  // What the binding holds open (threadsafe functions, async work) holds a
+  // Node host's event loop open until released; a dead binding releases
+  // nothing, so a fatal error lets go of it all.
+  let waiting = 0;
+  const increaseWaiting = context.increaseWaitingRequestCounter.bind(context);
+  const decreaseWaiting = context.decreaseWaitingRequestCounter.bind(context);
+  context.increaseWaitingRequestCounter = () => {
+    waiting++;
+    increaseWaiting();
+  };
+  context.decreaseWaitingRequestCounter = () => {
+    if (waiting === 0) return;
+    waiting--;
+    decreaseWaiting();
+  };
   const { napiModule } = instantiateNapiModuleSync(host.binding, {
     context,
     asyncWorkPoolSize: 0,