@nimbus-sh/worker 0.8.0 → 0.10.0

package/scripts/bundle-esbuild-wasm.mjs

@@ -10,8 +10,10 @@
  * than the supervisor's. The facet needs two pieces of esbuild-wasm
  * at module-load time:
  *
- *   1. The JS adapter (`esbuild-wasm/esm/browser.js`) — small, stays
- *      in the worker bundle as a string constant.
+ *   1. The JS adapter (`esbuild-wasm/esm/browser.js`, ~117 KiB), rewritten
+ *      into a function body. The supervisor already imports the same
+ *      build as a module for its own transforms, so the adapter is staged
+ *      beside the wasm rather than embedded a second time as a string.
  *
  *   2. The wasm binary (`esbuild.wasm`) — 12 MiB. Lives in the
  *      static-assets layer (Workers ASSETS binding) at
@@ -48,11 +50,15 @@
  * Output:
  *   src/esbuild-wasm-bundle.generated.ts
  *     export const ESBUILD_WASM_VERSION: string;
+ *     export const ESBUILD_WASM_ASSET_PATH: string;  // /_assets/esbuild-<version>.wasm
+ *     export const ESBUILD_JS_ASSET_PATH: string;    // /_assets/esbuild-<version>.js
  *     export const ESBUILD_WASM_SHA256: string;      // staged wasm digest
- *     export const ESBUILD_WASM_JS_FN_BODY: string;  // ESM browser source
+ *     export const ESBUILD_JS_SHA256: string;        // staged adapter digest
  *
  *   public/_assets/esbuild-<version>.wasm
  *     The raw wasm, picked up by the wrangler ASSETS binding.
+ *   public/_assets/esbuild-<version>.js
+ *     The adapter as a function body, same binding.
  *
  * Run via:
  *   node scripts/bundle-esbuild-wasm.mjs
@@ -144,8 +150,8 @@ async function main() {
   // The ASSETS binding picks up everything under `assets.directory`
   // (./public per wrangler.jsonc). We version-name the file so a
   // future esbuild-wasm bump produces a different asset filename and
-  // forces a fresh fetch (no stale-cache risk; cf. ESBUILD_WASM_ASSET_PATH
-  // in src/esbuild-wasm-bytes.ts).
+  // forces a fresh fetch (no stale-cache risk; the generated module names
+  // the path and src/runtime/esbuild-wasm-bytes.ts reads it).
   await fs.mkdir(OUT_ASSETS_DIR, { recursive: true });
   const assetName = `esbuild-${version}.wasm`;
   const assetOut = path.join(OUT_ASSETS_DIR, assetName);
@@ -155,53 +161,48 @@ async function main() {
   console.log(
     `[bundle-esbuild-wasm] copied esbuild.wasm → ${path.relative(ROOT, assetOut)} (${(wasmBytes.length / (1024 * 1024)).toFixed(1)} MiB)`,
   );
+  // The JS adapter is staged the same way. Facets splice it into their
+  // source as a string; the supervisor imports esbuild-wasm as a module
+  // for its own transforms, so carrying the adapter inline as well
+  // shipped it twice in the Worker bundle.
+  const jsAssetName = `esbuild-${version}.js`;
+  const jsAssetOut = path.join(OUT_ASSETS_DIR, jsAssetName);
+  const jsBytes = Buffer.from(jsFn, 'utf8');
+  await fs.writeFile(jsAssetOut, jsBytes);
+  const jsSha256 = createHash('sha256').update(jsBytes).digest('hex');
+  console.log(
+    `[bundle-esbuild-wasm] wrote JS adapter → ${path.relative(ROOT, jsAssetOut)} (${(jsBytes.length / 1024).toFixed(1)} KiB)`,
+  );
 
   // ── 3. Clean up any stale-versioned wasm assets in public/_assets/ ──
   // Keeps the deploy lean: a developer who bumped the esbuild-wasm
   // version locally won't accidentally ship two copies.
   for (const entry of await fs.readdir(OUT_ASSETS_DIR)) {
-    if (entry.startsWith('esbuild-') && entry.endsWith('.wasm') && entry !== assetName) {
+    if (entry.startsWith('esbuild-') && (entry.endsWith('.wasm') || entry.endsWith('.js'))
+      && entry !== assetName && entry !== jsAssetName) {
       const stale = path.join(OUT_ASSETS_DIR, entry);
       await fs.unlink(stale);
       console.log(`[bundle-esbuild-wasm] removed stale asset: ${entry}`);
     }
   }
 
-  // ── 4. Emit the JS-only generated TS module ──────────────────────────
-  // No ESBUILD_WASM_BASE64 export. The base64 string was 16 MiB and
-  // sat in supervisor module scope for the isolate's lifetime; A'.5
-  // moved the bytes to env.ASSETS to drop that residency.
+  // ── 4. Emit the generated TS module: versions and digests only ──────
+  // Neither artifact is inline. The base64 wasm string was 16 MiB and
+  // sat in supervisor module scope for the isolate's lifetime (A'.5
+  // moved it to env.ASSETS); the JS adapter followed so the bundle
+  // carries esbuild-wasm once, as the module the supervisor imports.
   const header = `/**
  * esbuild-wasm-bundle.generated.ts — AUTO-GENERATED by scripts/bundle-esbuild-wasm.mjs
  * DO NOT EDIT.
  *
- * Bundled esbuild-wasm ${version} adapter for use inside NimbusIsolatePool
- * isolates. Only the JS adapter is bundled here (~117 KiB); the wasm
- * binary itself lives in the static-assets layer at
- * public/_assets/esbuild-${version}.wasm and is fetched on demand by
- * src/esbuild-wasm-bytes.ts.
+ * Digests of the staged esbuild-wasm ${version} artifacts. Both live in
+ * the static-assets layer, public/_assets/esbuild-${version}.{wasm,js},
+ * and are fetched on demand and verified by src/runtime/esbuild-wasm-bytes.ts.
  *
- * Phase 2 A'.5 moved the base64 wasm string OUT of the supervisor
- * bundle (was 16 MiB UTF-8 → 21 MiB UTF-16 in JS string overhead).
- * If you're reading this and wondering where the bytes are: env.ASSETS.
- */
-
-export const ESBUILD_WASM_VERSION: string = ${JSON.stringify(version)};
-
-/**
- * SHA-256 of the staged public/_assets/esbuild-${version}.wasm bytes.
- * src/runtime/esbuild-wasm-bytes.ts verifies every fetch against it — the L2
- * (caches.default) tier included — before the bytes reach workerd's loader.
- */
-export const ESBUILD_WASM_SHA256: string = ${JSON.stringify(wasmSha256)};
-
-/**
- * Function body that, when wrapped in \`new Function(...)()\`, returns
- * the esbuild-wasm namespace (build, transform, initialize, ...).
- *
- * Source: node_modules/esbuild-wasm/esm/browser.js with the trailing
- * \`export { ... }\` rewritten to \`return { ... };\`. Why the browser
- * build:
+ * The JS adapter is node_modules/esbuild-wasm/esm/browser.js with the
+ * trailing \`export { ... }\` rewritten to \`return { ... };\`, so that
+ * \`new Function(body)()\` yields the esbuild namespace inside a facet.
+ * Why the browser build:
  *   - Pure ESM with no \`createRequire\`, no \`require("fs")\`, and
  *     no \`process.binding\` — safe to evaluate inside a workerd
  *     dynamic-worker isolate.
@@ -219,13 +220,28 @@ export const ESBUILD_WASM_SHA256: string = ${JSON.stringify(wasmSha256)};
  * patterns (createRequire / dynamic node-builtin require / process.binding)
  * and bails loud if a future release reintroduces them.
  */
-export const ESBUILD_WASM_JS_FN_BODY: string = ${JSON.stringify(jsFn)};
+export const ESBUILD_WASM_VERSION: string = ${JSON.stringify(version)};
+/**
+ * Where the artifacts live inside env.ASSETS. Version-named, so a bump
+ * lands under a new name and never serves a stale cache entry. Naming them
+ * here is what puts them under scripts/dist-integrity.mjs's staged-asset
+ * check, which discovers asset paths by value in generated modules.
+ */
+export const ESBUILD_WASM_ASSET_PATH: string = ${JSON.stringify('/_assets/' + assetName)};
+export const ESBUILD_JS_ASSET_PATH: string = ${JSON.stringify('/_assets/' + jsAssetName)};
+/**
+ * SHA-256 of the staged public/_assets/esbuild-${version}.wasm bytes.
+ * src/runtime/esbuild-wasm-bytes.ts verifies every fetch against it — the L2
+ * (caches.default) tier included — before the bytes reach workerd's loader.
+ */
+export const ESBUILD_WASM_SHA256: string = ${JSON.stringify(wasmSha256)};
+/** SHA-256 of the staged public/_assets/esbuild-${version}.js adapter, checked the same way. */
+export const ESBUILD_JS_SHA256: string = ${JSON.stringify(jsSha256)};
 `;
-
   await fs.writeFile(OUT_TS, header, 'utf8');
   const stat = await fs.stat(OUT_TS);
   console.log(
-    `[bundle-esbuild-wasm] wrote ${path.relative(ROOT, OUT_TS)} (${(stat.size / 1024).toFixed(1)} KiB — JS adapter only)`,
+    `[bundle-esbuild-wasm] wrote ${path.relative(ROOT, OUT_TS)} (${(stat.size / 1024).toFixed(1)} KiB — digests only)`,
   );
 }
 

package/scripts/bundle-facet-workers.mjs

@@ -47,12 +47,53 @@ import { build } from 'esbuild';
 import { writeFileSync, mkdirSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+// core's own parser dependency, reached through the workspace hoist; the
+// script runs under plain node at postinstall, so nothing here is TypeScript.
+import { parse } from 'acorn';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const root = join(__dirname, '..');
 const coreRoot = join(root, '..', 'core');
 const platformRoot = join(root, '..', 'platform');
 
+/**
+ * The bundle without its comments. esbuild keeps every comment that sits
+ * inside an expression or object literal, plus its own `// path` module
+ * markers, and nothing reads any of them: the strings are evaluated, never
+ * shown, mapped or `toString()`ed. The comment ranges come from a real
+ * parse of the output, so a `//` inside a string, template or regex is
+ * untouched; only comments go, and a line that held nothing but a comment
+ * goes with it. Every other character, identifier and line stays put.
+ */
+function withoutComments(text) {
+  const comments = [];
+  parse(text, { ecmaVersion: 'latest', sourceType: 'module', allowAwaitOutsideFunction: true, onComment: comments });
+  let out = '';
+  let cursor = 0;
+  for (const { start, end } of comments) {
+    if (start < cursor) continue;
+    const lineStart = text.lastIndexOf('\n', start - 1) + 1;
+    const lineEndAt = text.indexOf('\n', end);
+    const lineEnd = lineEndAt < 0 ? text.length : lineEndAt;
+    const leading = text.slice(lineStart, start);
+    const trailing = text.slice(end, lineEnd);
+    if (leading.trim() === '' && trailing.trim() === '') {
+      // The comment is the whole line (or lines): drop them, newline included.
+      out += text.slice(cursor, lineStart);
+      cursor = lineEndAt < 0 ? text.length : lineEnd + 1;
+    } else if (trailing.trim() === '') {
+      // Trailing comment: drop it and the blank that led to it.
+      out += text.slice(cursor, start).replace(/[ \t]+$/, '');
+      cursor = end;
+    } else {
+      // Mid-line block comment: one space keeps the tokens on either side apart.
+      out += `${text.slice(cursor, start)} `;
+      cursor = end;
+    }
+  }
+  return out + text.slice(cursor);
+}
+
 /**
  * Bundle one TS source into a self-contained ESM string suitable for
  * inlining as a facet preamble. Strips the leading `export` on
@@ -76,7 +117,7 @@ async function bundleAsPreamble(entryPath, label) {
   if (!result.outputFiles || result.outputFiles.length === 0) {
     throw new Error(`[bundle-facet-workers/${label}] esbuild produced no output`);
   }
-  let stripped = result.outputFiles[0].text;
+  let stripped = withoutComments(result.outputFiles[0].text);
   stripped = stripped.replace(/^export\s+(async\s+function|function|const|class)\b/gm, '$1');
   stripped = stripped.replace(/\n?export\s*\{[^}]*\}\s*;\s*$/g, '');
   return stripped;
@@ -112,7 +153,7 @@ async function bundleVirtualSocketKernel() {
   if (!result.outputFiles || result.outputFiles.length === 0) {
     throw new Error('[bundle-facet-workers/virtual-socket-kernel] esbuild produced no output');
   }
-  return result.outputFiles[0].text;
+  return withoutComments(result.outputFiles[0].text);
 }
 
 /**
@@ -126,10 +167,7 @@ const WASI_REQUIRED_TOP_LEVEL = [
   '__wasiMakeImports',
   '__wasiRunStart',
   '__wasiRunStartAsync',
-  '__wasiReadFilesB64',
   '__wasiAdoptSupervisor',
-  '__wasiDrainPersist',
-  '__wasiRevalidateFS',
   'fdTable',
 ];
 
@@ -168,7 +206,7 @@ async function bundleWasiInstance() {
   if (!result.outputFiles || result.outputFiles.length === 0) {
     throw new Error('[bundle-facet-workers/wasi-instance] esbuild produced no output');
   }
-  let src = result.outputFiles[0].text;
+  let src = withoutComments(result.outputFiles[0].text);
   // The body is spliced into another module; a re-export block there is a
   // syntax error, and callers append their own `export { … }`.
   src = src.replace(/^export\s+(async\s+function|function|const|let|var|class)\b/gm, '$1');
@@ -233,7 +271,7 @@ async function bundleBashRunner() {
   if (!result.outputFiles || result.outputFiles.length === 0) {
     throw new Error('[bundle-facet-workers/bash-runner] esbuild produced no output');
   }
-  const src = result.outputFiles[0].text;
+  const src = withoutComments(result.outputFiles[0].text);
   for (const name of ['__bashBoot', '__bashFeed']) {
     if (!new RegExp(`^\\s*globalThis\\.${name}\\s*=`, 'm').test(src)) {
       throw new Error(

package/scripts/bundle-runtime.mjs

@@ -7,7 +7,7 @@
  *   node scripts/bundle-runtime.mjs clang binji-2020 [--bucket nimbus-runtime-cache]
  *   node scripts/bundle-runtime.mjs python 0.29.4 [--bucket nimbus-runtime-cache]
  *   node scripts/bundle-runtime.mjs --pin-catalog        (read-only; publishes nothing)
- *   node scripts/bundle-runtime.mjs bash 5.2.37 --npm-package <dir>   (local; no R2)
+ *   node scripts/bundle-runtime.mjs bash 5.2.37-2 --npm-package <dir>   (local; no R2)
  *
  * `--npm-package` is the SECOND publisher for the same artifacts: it stages
  * exactly what the R2 path stages, composes the same manifest bytes, and lays
@@ -50,6 +50,7 @@ import { mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, openSync, c
 import { execFileSync, execSync, spawnSync } from 'node:child_process';
 import { tmpdir } from 'node:os';
 import { basename, dirname, join } from 'node:path';
+import { BASH_RUNNER } from '@nimbus-sh/core/runtime/os-contracts.js';
 
 const ACCOUNT = process.env.CLOUDFLARE_ACCOUNT_ID;
 // Resolve the wrangler bin from either the package-local or the
@@ -64,9 +65,14 @@ const PYODIDE_WORKERD_ADAPTER = JSON.parse(
 );
 
 const USAGE =
-  'usage: bundle-runtime.mjs <name> <version> [--bucket <bucket>]\n' +
+  'usage: bundle-runtime.mjs <name> <version> [--bucket <bucket>] [--keep-default]\n' +
   '       bundle-runtime.mjs <name> <version> --npm-package <dir>\n' +
-  '       bundle-runtime.mjs --pin-catalog [--bucket <bucket>]';
+  '       bundle-runtime.mjs --pin-catalog [--bucket <bucket>]\n' +
+  '\n' +
+  '  --keep-default  add the version to the catalog without making it the\n' +
+  '                  default: for a build whose runner contract a deployment\n' +
+  '                  still reading the catalog cannot bind yet. Re-run without\n' +
+  '                  the flag once every deployment can; the re-run is idempotent.';
 
 /** The bucket the deployed Worker's NIMBUS_RUNTIME_CACHE binding points at.
  *  Only a publish to THIS bucket may rewrite the catalog pin: the pin
@@ -78,6 +84,7 @@ const rawArgs = process.argv.slice(2);
 const positionalArgs = [];
 let BUCKET = process.env.NIMBUS_RUNTIME_BUCKET || PRODUCTION_BUCKET;
 let PIN_ONLY = false;
+let KEEP_DEFAULT = false;
 let NPM_OUT_DIR = null;
 for (let i = 0; i < rawArgs.length; i++) {
   const arg = rawArgs[i];
@@ -85,6 +92,10 @@ for (let i = 0; i < rawArgs.length; i++) {
     PIN_ONLY = true;
     continue;
   }
+  if (arg === '--keep-default') {
+    KEEP_DEFAULT = true;
+    continue;
+  }
   if (arg === '--npm-package') {
     NPM_OUT_DIR = rawArgs[++i];
     if (!NPM_OUT_DIR) {
@@ -226,10 +237,16 @@ const SPECS = {
   // `local_base`. The bash-runner aliases every applet name in
   // busybox.applets onto the busybox module so bash's PATH lookup finds
   // them in /bin and /usr/bin.
-  'bash/5.2.37': {
+  //
+  // The version is `<upstream>-<build>`: the build number advances with
+  // every rebuild, and the entrypoint names the runner contract the build
+  // was made for (BASH_RUNNER), so a deployment whose preamble predates it
+  // never binds it. 5.2.37 is build 1, published under `bash-runner`.
+  'bash/5.2.37-2': {
     license: 'GPL-3.0-or-later AND GPL-2.0-only',
     wasi_namespace: 'wasi_snapshot_preview1',
     local_base: '../wasm/bash',
+    auxiliary_bins: 'coreutils/busybox.applets',
     npm: {
       name: '@nimbus-sh/runtime-bash',
       summary: 'GNU bash 5.2.37 and BusyBox 1.37.0, cross-compiled to wasm32-wasi',
@@ -238,7 +255,7 @@ const SPECS = {
       { src: 'bash.async.wasm',           vfs: 'share/bash/bash.async.wasm' },
       { src: 'coreutils/busybox.wasm',    vfs: 'share/bash/coreutils/busybox.wasm' },
       { src: 'coreutils/busybox.applets', vfs: 'share/bash/coreutils/busybox.applets' },
-      { src: 'BIN_MARKER', vfs: 'bin/bash', mode: 'exec', runner: 'bash-runner', binName: 'bash' },
+      { src: 'BIN_MARKER', vfs: 'bin/bash', mode: 'exec', runner: BASH_RUNNER, binName: 'bash' },
     ],
     synthetic_files: {
       'BIN_MARKER': Buffer.from(
@@ -427,6 +444,15 @@ if (!spec) {
   console.error(`known: ${Object.keys(SPECS).join(', ')}`);
   process.exit(2);
 }
+if (spec.auxiliary_bins) {
+  const list = new URL(`${spec.local_base}/${spec.auxiliary_bins}`, import.meta.url);
+  const names = readFileSync(list, 'utf8').split('\n').filter(Boolean);
+  spec.synthetic_files.NIMBUS_AUX_BIN = Buffer.from('Nimbus WASI multicall entry\n');
+  for (const name of names) {
+    if (name.includes('/') || name === '.' || name === '..') throw new Error(`Invalid multicall entry: ${name}`);
+    spec.files.push({ src: 'NIMBUS_AUX_BIN', vfs: `bin/${name}`, mode: 'exec' });
+  }
+}
 // Checked before anything is staged: which runtimes we publish to npm is a
 // decision, and a spec that has not made it should cost nothing to find out.
 if (NPM_OUT_DIR) {
@@ -617,8 +643,9 @@ if (spec.ingest_only) {
     license: spec.license,
   };
   // Update default to the just-uploaded version (idempotent — if it was
-  // already the default, no-op).
-  catalog.runtimes[RUNTIME].default = VERSION;
+  // already the default, no-op), unless the publisher asked to hold it for
+  // deployments that cannot bind this build yet.
+  if (!KEEP_DEFAULT) catalog.runtimes[RUNTIME].default = VERSION;
 
   // The catalog indexes every runtime, so an update that drops one is a
   // publish that unregisters somebody else's. Cheap to assert, and it catches
@@ -644,7 +671,7 @@ if (spec.ingest_only) {
   console.log(`\n[bundle-runtime] DONE`);
   console.log(`[bundle-runtime] uploaded ${downloaded.length} files (${totalMb} MiB) for ${RUNTIME}@${VERSION}`);
   console.log(`[bundle-runtime] manifest:  r2://${BUCKET}/${manifestR2Key}`);
-  console.log(`[bundle-runtime] catalog:   r2://${BUCKET}/${catalogR2Key}`);
+  console.log(`[bundle-runtime] catalog:   r2://${BUCKET}/${catalogR2Key} (default ${catalog.runtimes[RUNTIME].default})`);
 }
 
 /**