@nimbus-sh/worker 0.1.0

package/scripts/bundle-esbuild-wasm.mjs

@@ -0,0 +1,244 @@
+#!/usr/bin/env node
+/**
+ * bundle-esbuild-wasm.mjs — package esbuild-wasm for facet consumption.
+ *
+ * Why this exists
+ * ───────────────
+ * Pre-bundling npm packages (the `Pre-bundling N modules…` step in
+ * src/npm/installer.ts) runs inside NimbusLoaderPool isolates so each
+ * `esbuild.build()` allocation hits a fresh 128 MiB envelope rather
+ * than the supervisor's. The facet needs two pieces of esbuild-wasm
+ * at module-load time:
+ *
+ *   1. The JS adapter (`esbuild-wasm/esm/browser.js`) — small, stays
+ *      in the worker bundle as a string constant.
+ *
+ *   2. The wasm binary (`esbuild.wasm`) — 12 MiB. Lives in the
+ *      static-assets layer (Workers ASSETS binding) at
+ *      `public/_assets/esbuild-<version>.wasm` and is fetched on
+ *      demand by the supervisor at pool-construction time. NEVER
+ *      bundled into the worker code object — keeping the bytes out
+ *      of the supervisor's static baseline drops ~21 MiB of resident
+ *      heap (verified via src/observability/heap-estimate.ts).
+ *
+ * Pre-A'.5 design (this script's prior form) embedded the wasm bytes
+ * as a base64 string in src/esbuild-wasm-bundle.generated.ts. That
+ * had two problems:
+ *   • The 16 MiB base64 string lived in supervisor module scope for
+ *     the lifetime of the isolate (~21 MiB UTF-16 overhead).
+ *   • Decoding to a Uint8Array allocated another 12 MiB which the
+ *     prior esbuild-wasm-bytes.ts cached for the same lifetime.
+ * Combined: ~37 MiB resident in supervisor for esbuild that almost
+ * never used it directly. Phase 2 A'.5 moved that out.
+ *
+ * Workarounds tried & rejected (kept here as institutional memory):
+ *   1. The default CJS entry (`lib/main.js`) calls
+ *      `createRequire(import.meta.url)('fs')` at module init.
+ *      `nodejs_compat` only satisfies static `import 'node:fs'`,
+ *      so dynamic-require throws on dynamic-worker isolates.
+ *   2. The `import esbuildWasmUrl from 'esbuild-wasm/esbuild.wasm'`
+ *      asset binding is a wrangler bundling feature — facets built
+ *      from inline `modules:` strings have no asset resolver, so
+ *      that import fails in the facet.
+ *   3. Fetching the wasm from R2 at facet boot — adds an RPC hop
+ *      per dispatch, costs more than the in-supervisor fetch we do
+ *      now (which happens once per pool construction, not per
+ *      dispatch).
+ *
+ * Output:
+ *   src/esbuild-wasm-bundle.generated.ts
+ *     export const ESBUILD_WASM_VERSION: string;
+ *     export const ESBUILD_WASM_JS_FN_BODY: string;  // ESM browser source
+ *
+ *   public/_assets/esbuild-<version>.wasm
+ *     The raw wasm, picked up by the wrangler ASSETS binding.
+ *
+ * Run via:
+ *   node scripts/bundle-esbuild-wasm.mjs
+ *   (also wired into package.json predev/predeploy/postinstall)
+ *
+ * If a future esbuild-wasm release breaks `esm/browser.js` (e.g.
+ * reintroduces a dynamic require), this script bails loud rather
+ * than shipping a broken bundle to prod.
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+// Walk up looking for node_modules/esbuild-wasm. Handles both
+// packages/worker/node_modules/esbuild-wasm (legacy single-pkg layout)
+// and ../../node_modules/esbuild-wasm (monorepo hoist).
+function resolvePkgDir(start, pkgName) {
+  let dir = start;
+  while (true) {
+    const candidate = path.join(dir, 'node_modules', pkgName);
+    try {
+      // Throws if missing; cheap stat.
+      // eslint-disable-next-line n/no-sync
+      require('node:fs').statSync(candidate);
+      return candidate;
+    } catch { /* fall through */ }
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(`Cannot locate node_modules/${pkgName} starting at ${start}`);
+}
+const { createRequire } = await import('node:module');
+const require = createRequire(import.meta.url);
+const PKG_DIR = resolvePkgDir(ROOT, 'esbuild-wasm');
+const JS_SRC = path.join(PKG_DIR, 'esm', 'browser.js');
+const WASM_SRC = path.join(PKG_DIR, 'esbuild.wasm');
+const PKG_JSON = path.join(PKG_DIR, 'package.json');
+const OUT_TS = path.join(ROOT, 'src', 'esbuild-wasm-bundle.generated.ts');
+const OUT_ASSETS_DIR = path.join(ROOT, 'public', '_assets');
+
+/**
+ * Patterns that, if present in the JS source, mean the browser build
+ * has been changed and is no longer safe to run inside a dynamic
+ * worker. The script bails rather than silently shipping a broken
+ * bundle to prod.
+ */
+const FORBIDDEN_PATTERNS = [
+  /createRequire\s*\(/,
+  /require\(["'](fs|path|os|child_process|module|crypto|stream)["']\)/,
+  /process\.binding\s*\(/,
+];
+
+async function main() {
+  const pkgJson = JSON.parse(await fs.readFile(PKG_JSON, 'utf8'));
+  const version = pkgJson.version;
+  console.log(`[bundle-esbuild-wasm] version: ${version}`);
+
+  // ── 1. Read & rewrite the ESM browser entry ─────────────────────────
+  // We rewrite the trailing `export { ... }` into `return { ... };` so
+  // the entire file becomes the body of a function. The facet
+  // evaluates it via `new Function(body)()`, which workerd permits
+  // inside dynamic workers at startup-time (precedent:
+  // src/facet-manager.ts).
+  const jsRaw = await fs.readFile(JS_SRC, 'utf8');
+  console.log(`[bundle-esbuild-wasm] esm/browser.js: ${(jsRaw.length / 1024).toFixed(1)} KiB`);
+
+  for (const re of FORBIDDEN_PATTERNS) {
+    if (re.test(jsRaw)) {
+      throw new Error(
+        `[bundle-esbuild-wasm] esm/browser.js contains forbidden pattern ${re}\n` +
+          `   The browser build is no longer safe for facet consumption.\n` +
+          `   Update this script to patch the offending call site or pin\n` +
+          `   esbuild-wasm to the prior version.`,
+      );
+    }
+  }
+
+  const exportRe = /\nexport\s*\{([\s\S]+?)\};?\s*$/;
+  const m = jsRaw.match(exportRe);
+  if (!m) {
+    throw new Error(
+      `[bundle-esbuild-wasm] could not locate trailing 'export {…}' block\n` +
+        `   in esm/browser.js — esbuild-wasm may have changed shape.`,
+    );
+  }
+  const exportBody = m[1];
+  const objectMembers = exportBody
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean)
+    .map((entry) => {
+      const asMatch = entry.match(/^(\S+)\s+as\s+(\S+)$/);
+      if (asMatch) return `${asMatch[2]}: ${asMatch[1]}`;
+      return entry;
+    })
+    .join(', ');
+  const jsFn = jsRaw.replace(exportRe, '\n') + `\nreturn { ${objectMembers} };\n`;
+  console.log(
+    `[bundle-esbuild-wasm] rewrote ${exportBody.split(',').length} exports → object-return shape`,
+  );
+
+  // ── 2. Stage the wasm into public/_assets/ ──────────────────────────
+  // The ASSETS binding picks up everything under `assets.directory`
+  // (./public per wrangler.jsonc). We version-name the file so a
+  // future esbuild-wasm bump produces a different asset filename and
+  // forces a fresh fetch (no stale-cache risk; cf. ESBUILD_WASM_ASSET_PATH
+  // in src/esbuild-wasm-bytes.ts).
+  await fs.mkdir(OUT_ASSETS_DIR, { recursive: true });
+  const assetName = `esbuild-${version}.wasm`;
+  const assetOut = path.join(OUT_ASSETS_DIR, assetName);
+  const wasmBytes = await fs.readFile(WASM_SRC);
+  await fs.writeFile(assetOut, wasmBytes);
+  console.log(
+    `[bundle-esbuild-wasm] copied esbuild.wasm → ${path.relative(ROOT, assetOut)} (${(wasmBytes.length / (1024 * 1024)).toFixed(1)} MiB)`,
+  );
+
+  // ── 3. Clean up any stale-versioned wasm assets in public/_assets/ ──
+  // Keeps the deploy lean: a developer who bumped the esbuild-wasm
+  // version locally won't accidentally ship two copies.
+  for (const entry of await fs.readdir(OUT_ASSETS_DIR)) {
+    if (entry.startsWith('esbuild-') && entry.endsWith('.wasm') && entry !== assetName) {
+      const stale = path.join(OUT_ASSETS_DIR, entry);
+      await fs.unlink(stale);
+      console.log(`[bundle-esbuild-wasm] removed stale asset: ${entry}`);
+    }
+  }
+
+  // ── 4. Emit the JS-only generated TS module ──────────────────────────
+  // No ESBUILD_WASM_BASE64 export. The base64 string was 16 MiB and
+  // sat in supervisor module scope for the isolate's lifetime; A'.5
+  // moved the bytes to env.ASSETS to drop that residency.
+  const header = `/**
+ * esbuild-wasm-bundle.generated.ts — AUTO-GENERATED by scripts/bundle-esbuild-wasm.mjs
+ * DO NOT EDIT.
+ *
+ * Bundled esbuild-wasm ${version} adapter for use inside NimbusLoaderPool
+ * isolates. Only the JS adapter is bundled here (~117 KiB); the wasm
+ * binary itself lives in the static-assets layer at
+ * public/_assets/esbuild-${version}.wasm and is fetched on demand by
+ * src/esbuild-wasm-bytes.ts.
+ *
+ * Phase 2 A'.5 moved the base64 wasm string OUT of the supervisor
+ * bundle (was 16 MiB UTF-8 → 21 MiB UTF-16 in JS string overhead).
+ * If you're reading this and wondering where the bytes are: env.ASSETS.
+ */
+
+export const ESBUILD_WASM_VERSION: string = ${JSON.stringify(version)};
+
+/**
+ * Function body that, when wrapped in \`new Function(...)()\`, returns
+ * the esbuild-wasm namespace (build, transform, initialize, ...).
+ *
+ * Source: node_modules/esbuild-wasm/esm/browser.js with the trailing
+ * \`export { ... }\` rewritten to \`return { ... };\`. Why the browser
+ * build:
+ *   - Pure ESM with no \`createRequire\`, no \`require("fs")\`, and
+ *     no \`process.binding\` — safe to evaluate inside a workerd
+ *     dynamic-worker isolate.
+ *   - The supported public surface for "esbuild inside a worker" is
+ *     exactly this build.
+ *
+ * Why \`new Function(body)()\` (rather than \`import('data:...')\`):
+ *   - data: URL imports across workerd versions are not contractual
+ *     and we don't want a deploy that worked yesterday to silently
+ *     break tomorrow when the runtime is upgraded.
+ *   - new Function is documented as supported inside dynamic workers
+ *     (src/facet-manager.ts has been doing this since 2025).
+ *
+ * The bundle script verifies the file does NOT contain the forbidden
+ * patterns (createRequire / dynamic node-builtin require / process.binding)
+ * and bails loud if a future release reintroduces them.
+ */
+export const ESBUILD_WASM_JS_FN_BODY: string = ${JSON.stringify(jsFn)};
+`;
+
+  await fs.writeFile(OUT_TS, header, 'utf8');
+  const stat = await fs.stat(OUT_TS);
+  console.log(
+    `[bundle-esbuild-wasm] wrote ${path.relative(ROOT, OUT_TS)} (${(stat.size / 1024).toFixed(1)} KiB — JS adapter only)`,
+  );
+}
+
+main().catch((e) => {
+  console.error('[bundle-esbuild-wasm] failed:', e);
+  process.exit(1);
+});

package/scripts/bundle-facet-workers.mjs

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+/**
+ * bundle-facet-workers.mjs — Produce the preamble string that
+ * NimbusFacetPool injects into dynamic workers.
+ *
+ * WHY this exists:
+ *   NimbusFacetPool's user functions get wrapped by cloudflare-parallel's
+ *   codegen into a module source. Inside that module the user function
+ *   can reference closure-variables only through the `context` option,
+ *   which is JSON-only — you cannot pass a helper function through it.
+ *
+ *   For the npm install facet we need the streaming tar parser
+ *   (src/npm/tarball-stream.ts) available as a top-level named export
+ *   the user function can call. We esbuild-bundle that source file into
+ *   a self-contained ES module string, and NimbusFacetPool's `preamble`
+ *   option splices it into the generated module between the
+ *   WorkerEntrypoint import and the user function.
+ *
+ * Output:
+ *   src/loaders/generated-workers.ts — exports
+ *       TAR_STREAM_PREAMBLE: string
+ *       TAR_STREAM_PREAMBLE_SIZE: number
+ *       W7_FRAME_PREAMBLE: string         (W7 — streaming bulk-write encoder)
+ *       W7_FRAME_PREAMBLE_SIZE: number
+ *
+ * Runs as a postinstall + predev + predeploy step via package.json.
+ */
+
+import { build } from 'esbuild';
+import { writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const root = join(__dirname, '..');
+
+/**
+ * Bundle one TS source into a self-contained ESM string suitable for
+ * inlining as a facet preamble. Strips the leading `export` on
+ * declarations and the aggregate `export { ... };` block so the
+ * blob is inlinable into another module without re-export errors.
+ */
+async function bundleAsPreamble(entryPath, label) {
+  const result = await build({
+    entryPoints: [entryPath],
+    bundle: true,
+    format: 'esm',
+    target: 'esnext',
+    platform: 'neutral',
+    write: false,
+    logLevel: 'warning',
+    legalComments: 'none',
+    // Strip TypeScript-only imports (e.g. `import type {…}`) — esbuild
+    // already drops these, but leave the option default.
+  });
+  if (!result.outputFiles || result.outputFiles.length === 0) {
+    throw new Error(`[bundle-facet-workers/${label}] esbuild produced no output`);
+  }
+  let stripped = result.outputFiles[0].text;
+  stripped = stripped.replace(/^export\s+(async\s+function|function|const|class)\b/gm, '$1');
+  stripped = stripped.replace(/\n?export\s*\{[^}]*\}\s*;\s*$/g, '');
+  return stripped;
+}
+
+async function main() {
+  // 1. Tar-parser preamble (existing W2.5/W4 hot-path helpers).
+  const tarStripped = await bundleAsPreamble(
+    join(root, 'src', 'npm', 'tarball-stream.ts'),
+    'tar-stream',
+  );
+
+  // 2. W7 frame encoder preamble. The npm-install-batch-facet calls
+  //    encodeWriteBatchStream() to wrap its writeBatch payload as a
+  //    type:'bytes' ReadableStream, then passes the stream to
+  //    env.SUPERVISOR.writeBatchStream(). Without this preamble the
+  //    facet has no access to the encoder symbol (cloudflare-parallel
+  //    serialises via fn.toString() — no runtime imports).
+  //
+  //    The W7-frame module imports a TypeScript type from sqlite-vfs,
+  //    which esbuild's type-stripping handles transparently. The
+  //    runtime output has no imports.
+  const w7Stripped = await bundleAsPreamble(
+    join(root, 'src', '_shared', 'w7-frame.ts'),
+    'w7-frame',
+  );
+
+  const tarEncoded = JSON.stringify(tarStripped);
+  const w7Encoded = JSON.stringify(w7Stripped);
+  const outPath = join(root, 'src', 'loaders', 'generated-workers.ts');
+
+  const tsWrapper = [
+    '/**',
+    ' * generated-workers.ts — AUTO-GENERATED. DO NOT EDIT.',
+    ' *',
+    ' * Produced by scripts/bundle-facet-workers.mjs from:',
+    ' *   - src/npm/tarball-stream.ts (streaming tar primitives)',
+    ' *   - src/_shared/w7-frame.ts   (W7 streaming bulk-write encoder)',
+    ' *',
+    ' * Consumed by src/loaders/loader-pool.ts callers via the `preamble`',
+    ' * option. The preamble is injected at the top of every generated',
+    ' * worker module so user functions can reference the exported',
+    ' * helpers by name.',
+    ' *',
+    ' * Tar-stream symbols: parseTarHeader, streamTarEntries,',
+    ' *   readableStreamToAsyncIterable, MAX_FILE_BYTES.',
+    ' * W7-frame symbols:   encodeWriteBatchStream, decodeWriteBatchStream,',
+    ' *   W7_MAGIC, W7_TRAILER.',
+    ' *',
+    ` * Generated at: ${new Date().toISOString()}`,
+    ` * Tar size: ${(tarStripped.length / 1024).toFixed(2)} KiB`,
+    ` * W7 size:  ${(w7Stripped.length / 1024).toFixed(2)} KiB`,
+    ' */',
+    '',
+    `export const TAR_STREAM_PREAMBLE: string = ${tarEncoded};`,
+    '',
+    `export const TAR_STREAM_PREAMBLE_SIZE: number = ${tarStripped.length};`,
+    '',
+    `export const W7_FRAME_PREAMBLE: string = ${w7Encoded};`,
+    '',
+    `export const W7_FRAME_PREAMBLE_SIZE: number = ${w7Stripped.length};`,
+    '',
+  ].join('\n');
+
+  mkdirSync(dirname(outPath), { recursive: true });
+  writeFileSync(outPath, tsWrapper);
+
+  console.log(
+    `[bundle-facet-workers] wrote ${outPath} ` +
+    `(tar=${(tarStripped.length / 1024).toFixed(2)} KiB, ` +
+    `w7=${(w7Stripped.length / 1024).toFixed(2)} KiB)`,
+  );
+}
+
+main().catch((e) => {
+  console.error('[bundle-facet-workers] FAILED:', e?.message || e);
+  process.exitCode = 1;
+});

package/scripts/bundle-git.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * bundle-git.mjs — Pre-bundle isomorphic-git for the git-network-facet.
+ *
+ * WHY this exists:
+ *   The git-network-facet runs inside a dynamic worker loaded via
+ *   env.LOADER.load(). Dynamic workers have NO access to node_modules —
+ *   they only see modules explicitly passed in the `modules` object.
+ *
+ *   The main Worker's bundler (wrangler) inlines isomorphic-git into the
+ *   supervisor, but that bundle is unreachable from the facet isolate.
+ *
+ *   This script bundles isomorphic-git + isomorphic-git/http/web into ONE
+ *   ESM string that we can pass as a module to LOADER.load(). The facet
+ *   imports it via `import('./isomorphic-git.js')`.
+ *
+ * Output:
+ *   src/git-bundle.generated.ts — exports GIT_BUNDLE_CODE: string
+ *
+ * Runs as a postinstall step AND on every `bun run dev` / `bun run deploy`
+ * via the "bundle:git" npm script.
+ */
+
+import { build } from 'esbuild';
+import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const root = join(__dirname, '..');
+
+// Entry that aggregates both namespaces into a single ESM bundle.
+const ENTRY_CONTENTS = [
+  "export * as git from 'isomorphic-git';",
+  "export * as gitHttp from 'isomorphic-git/http/web';",
+].join('\n');
+
+const ENTRY_FILE = join(root, '.git-bundle-entry.js');
+
+async function main() {
+  try {
+    writeFileSync(ENTRY_FILE, ENTRY_CONTENTS);
+
+    const result = await build({
+      entryPoints: [ENTRY_FILE],
+      bundle: true,
+      format: 'esm',
+      target: 'esnext',
+      platform: 'browser',
+      conditions: ['worker', 'browser', 'import'],
+      // Map bare node built-ins → node: prefixed (works with nodejs_compat).
+      alias: {
+        crypto: 'node:crypto',
+      },
+      // Anything else that might leak: leave as external. nodejs_compat
+      // provides these at runtime in the dynamic worker.
+      external: [
+        'node:*',
+        'cloudflare:workers',
+      ],
+      write: false,
+      logLevel: 'warning',
+    });
+
+    if (!result.outputFiles || result.outputFiles.length === 0) {
+      throw new Error('esbuild produced no output');
+    }
+
+    const code = result.outputFiles[0].text;
+
+    // Emit as a TS module that exports the bundle as a string constant.
+    // Using String.raw-equivalent: JSON-encode to preserve backticks/$/etc.
+    const outPath = join(root, 'src', 'git-bundle.generated.ts');
+
+    // We JSON.stringify the code to produce a valid JS string literal.
+    // Then wrap in a TS const export. The supervisor imports this at build.
+    const encoded = JSON.stringify(code);
+
+    const tsWrapper = [
+      '/**',
+      ' * git-bundle.generated.ts — AUTO-GENERATED. DO NOT EDIT.',
+      ' *',
+      ' * Produced by scripts/bundle-git.mjs from:',
+      " *   - isomorphic-git (github:AshishKumar4/cf-git)",
+      " *   - isomorphic-git/http/web",
+      ' *',
+      ' * Consumed by git-network-facet.ts: passed to LOADER.load()\'s',
+      ' * `modules` record so the facet can `import` isomorphic-git.',
+      ' *',
+      ` * Generated at: ${new Date().toISOString()}`,
+      ` * Size: ${(code.length / 1024).toFixed(1)} KiB`,
+      ' */',
+      '',
+      `export const GIT_BUNDLE_CODE: string = ${encoded};`,
+      '',
+      `export const GIT_BUNDLE_SIZE = ${code.length};`,
+      '',
+    ].join('\n');
+
+    mkdirSync(dirname(outPath), { recursive: true });
+    writeFileSync(outPath, tsWrapper);
+
+    console.log(`[bundle-git] wrote ${outPath} (${(code.length / 1024).toFixed(1)} KiB bundled)`);
+  } finally {
+    // Clean up the temp entry file
+    try {
+      if (existsSync(ENTRY_FILE)) {
+        const { unlinkSync } = await import('node:fs');
+        unlinkSync(ENTRY_FILE);
+      }
+    } catch {}
+  }
+}
+
+main().catch((e) => {
+  console.error('[bundle-git] FAILED:', e?.message || e);
+  process.exitCode = 1;
+});