@scelar/nodepod 1.0.0

package/src/module-transformer.ts

@@ -0,0 +1,368 @@
+// ESM-to-CJS bulk transformer (esbuild-wasm). Used at install time to
+// pre-convert packages so synchronous require() works at runtime.
+// Transforms are offloaded to Web Workers when available.
+
+import type { MemoryVolume } from "./memory-volume";
+import { CDN_ESBUILD_ESM, CDN_ESBUILD_BINARY, cdnImport } from "./constants/cdn-urls";
+import { offload, taskId, TaskPriority } from "./threading/offload";
+import type { TransformResult } from "./threading/offload-types";
+
+const inBrowser = typeof window !== "undefined";
+
+// load and init esbuild-wasm from CDN (reuses existing instance)
+export async function prepareTransformer(): Promise<void> {
+  if (!inBrowser) {
+    return;
+  }
+
+  if (window.__esbuildEngine) {
+    return;
+  }
+
+  if (window.__esbuildReady) {
+    return window.__esbuildReady;
+  }
+
+  window.__esbuildReady = (async () => {
+    try {
+      const loaded = await cdnImport(CDN_ESBUILD_ESM);
+      const engine = loaded.default || loaded;
+
+      try {
+        await engine.initialize({ wasmURL: CDN_ESBUILD_BINARY });
+      } catch (initErr) {
+        if (
+          initErr instanceof Error &&
+          initErr.message.includes('Cannot call "initialize" more than once')
+        ) {
+          /* already initialized, ignore */
+        } else {
+          throw initErr;
+        }
+      }
+
+      window.__esbuildEngine = engine;
+    } catch (err) {
+      window.__esbuildReady = undefined;
+      throw err;
+    }
+  })();
+
+  return window.__esbuildReady;
+}
+
+export function isTransformerLoaded(): boolean {
+  if (!inBrowser) return true;
+  return window.__esbuildEngine !== undefined;
+}
+
+function containsJsx(source: string): boolean {
+  if (/<[A-Z][a-zA-Z0-9.]*[\s/>]/.test(source)) return true;
+  if (/<\/[a-zA-Z]/.test(source)) return true;
+  if (/\/>/.test(source)) return true;
+  if (/<>|<\/>/.test(source)) return true;
+  if (/React\.createElement\b/.test(source)) return true;
+  if (/jsx\(|jsxs\(|jsxDEV\(/.test(source)) return true;
+  return false;
+}
+
+function detectLoader(
+  filePath: string,
+  source: string,
+): "js" | "jsx" | "ts" | "tsx" {
+  if (filePath.endsWith(".jsx")) return "jsx";
+  if (filePath.endsWith(".ts")) return "ts";
+  if (filePath.endsWith(".tsx")) return "tsx";
+  if (filePath.endsWith(".mjs")) return "js";
+  if (containsJsx(source)) return "jsx";
+  return "js";
+}
+
+// convert a single file ESM->CJS via offloaded esbuild.transform()
+export async function convertFile(
+  source: string,
+  filePath: string,
+): Promise<string> {
+  if (!inBrowser) return source;
+
+  const result: TransformResult = await offload({
+    type: "transform",
+    id: taskId(),
+    source,
+    filePath,
+    options: {
+      loader: detectLoader(filePath, source),
+      format: "cjs",
+      target: "esnext",
+      platform: "neutral",
+      define: {
+        "import.meta.url": "import_meta.url",
+        "import.meta.dirname": "import_meta.dirname",
+        "import.meta.filename": "import_meta.filename",
+        "import.meta": "import_meta",
+      },
+    },
+    priority: TaskPriority.NORMAL,
+  });
+
+  // patch dynamic imports of node builtins to synchronous require()
+  let code = result.code;
+  code = code.replace(
+    /\bimport\s*\(\s*["']node:([^"']+)["']\s*\)/g,
+    'Promise.resolve(require("node:$1"))',
+  );
+
+  const coreModules = [
+    "assert", "buffer", "child_process", "cluster", "crypto",
+    "dgram", "dns", "events", "fs", "http", "http2", "https",
+    "net", "os", "path", "perf_hooks", "querystring", "readline",
+    "stream", "string_decoder", "timers", "tls", "url", "util",
+    "v8", "vm", "worker_threads", "zlib", "async_hooks",
+    "inspector", "module",
+  ];
+  for (const mod of coreModules) {
+    const re = new RegExp(`\\bimport\\s*\\(\\s*["']${mod}["']\\s*\\)`, "g");
+    code = code.replace(re, `Promise.resolve(require("${mod}"))`);
+  }
+
+  return code;
+}
+
+// direct main-thread esbuild transform (no worker offloading).
+// used as the fallback path to avoid circular offload -> convertFile -> offload.
+export async function convertFileDirect(
+  source: string,
+  filePath: string,
+): Promise<string> {
+  if (!inBrowser) return source;
+
+  if (!window.__esbuildEngine) await prepareTransformer();
+  const engine = window.__esbuildEngine;
+  if (!engine) throw new Error("esbuild engine not available");
+
+  const loader = detectLoader(filePath, source);
+
+  try {
+    const output = await engine.transform(source, {
+      loader,
+      format: "cjs",
+      target: "esnext",
+      platform: "neutral",
+      define: {
+        "import.meta.url": "import_meta.url",
+        "import.meta.dirname": "import_meta.dirname",
+        "import.meta.filename": "import_meta.filename",
+        "import.meta": "import_meta",
+      },
+    });
+    return output.code;
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? err.message : String(err);
+
+    // retry with alternative loaders
+    if (loader === "js" || loader === "jsx") {
+      const fallbacks = loader === "js" ? ["jsx", "tsx", "ts"] : ["tsx"];
+      for (const fallbackLoader of fallbacks) {
+        try {
+          const output = await engine.transform(source, {
+            loader: fallbackLoader as "jsx" | "tsx" | "ts",
+            format: "cjs",
+            target: "esnext",
+            platform: "neutral",
+            define: {
+              "import.meta.url": "import_meta.url",
+              "import.meta.dirname": "import_meta.dirname",
+              "import.meta.filename": "import_meta.filename",
+              "import.meta": "import_meta",
+            },
+          });
+          return output.code;
+        } catch {
+        }
+      }
+    }
+
+    if (msg.includes("Top-level await")) {
+      try {
+        const output = await engine.transform(source, {
+          loader,
+          format: "esm",
+          target: "esnext",
+          platform: "neutral",
+          define: {
+            "import.meta.url": "import_meta.url",
+            "import.meta.dirname": "import_meta.dirname",
+            "import.meta.filename": "import_meta.filename",
+            "import.meta": "import_meta",
+          },
+        });
+        return output.code;
+      } catch {
+        return source;
+      }
+    }
+    return source;
+  }
+}
+
+function requiresConversion(filePath: string, source: string): boolean {
+  // .mjs must stay ESM in the VFS -- tools read them directly. the runtime
+  // handles ESM->CJS on-the-fly for require().
+  if (filePath.endsWith(".mjs")) return false;
+  if (filePath.endsWith(".cjs")) return false;
+  return (
+    /\bimport\s+[\w{*'"]/m.test(source) ||
+    /\bexport\s+(?:default|const|let|var|function|class|{|\*)/m.test(source) ||
+    /\bimport\.meta\b/.test(source)
+  );
+}
+
+function needsBuiltinPatching(source: string): boolean {
+  return (
+    /\bimport\s*\(\s*["']node:/.test(source) ||
+    /\bimport\s*\(\s*["'](fs|path|http|https|net|url|util|events|stream|os|crypto)["']/.test(
+      source,
+    )
+  );
+}
+
+function patchBuiltinImports(source: string): string {
+  let patched = source;
+  patched = patched.replace(
+    /\bimport\s*\(\s*["']node:([^"']+)["']\s*\)/g,
+    'Promise.resolve(require("node:$1"))',
+  );
+  const builtins = [
+    "assert",
+    "buffer",
+    "child_process",
+    "cluster",
+    "crypto",
+    "dgram",
+    "dns",
+    "events",
+    "fs",
+    "http",
+    "http2",
+    "https",
+    "net",
+    "os",
+    "path",
+    "perf_hooks",
+    "querystring",
+    "readline",
+    "stream",
+    "string_decoder",
+    "timers",
+    "tls",
+    "url",
+    "util",
+    "v8",
+    "vm",
+    "worker_threads",
+    "zlib",
+    "async_hooks",
+    "inspector",
+    "module",
+  ];
+  for (const b of builtins) {
+    patched = patched.replace(
+      new RegExp(`\\bimport\\s*\\(\\s*["']${b}["']\\s*\\)`, "g"),
+      `Promise.resolve(require("${b}"))`,
+    );
+  }
+  return patched;
+}
+
+function listJsFiles(vol: MemoryVolume, dir: string): string[] {
+  const result: string[] = [];
+  try {
+    for (const name of vol.readdirSync(dir)) {
+      const full = dir + "/" + name;
+      try {
+        const info = vol.statSync(full);
+        if (info.isDirectory()) {
+          if (name !== "node_modules") result.push(...listJsFiles(vol, full));
+        } else if (
+          name.endsWith(".js") ||
+          name.endsWith(".mjs") ||
+          name.endsWith(".jsx")
+        ) {
+          result.push(full);
+        }
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return result;
+}
+
+function isEsmPackage(vol: MemoryVolume, packageDir: string): boolean {
+  try {
+    const pkgPath = packageDir + "/package.json";
+    if (vol.existsSync(pkgPath)) {
+      const raw = vol.readFileSync(pkgPath, "utf8");
+      const pkg = JSON.parse(raw);
+      return pkg.type === "module";
+    }
+  } catch {
+  }
+  return false;
+}
+
+// convert all ESM files in a package directory to CJS
+export async function convertPackage(
+  vol: MemoryVolume,
+  packageDir: string,
+  onProgress?: (msg: string) => void,
+): Promise<number> {
+  let converted = 0;
+  const files = listJsFiles(vol, packageDir);
+  onProgress?.(`  Converting ${files.length} files in ${packageDir}...`);
+
+  // "type":"module" packages stay as ESM in VFS -- tools like Vite read them
+  // directly and expect ESM syntax. require() handles conversion at runtime.
+  const esmPackage = isEsmPackage(vol, packageDir);
+
+  const BATCH = 50;
+  for (let i = 0; i < files.length; i += BATCH) {
+    const batch = files.slice(i, i + BATCH);
+    await Promise.all(
+      batch.map(async (fp) => {
+        try {
+          const src = vol.readFileSync(fp, "utf8");
+
+          if (esmPackage) {
+            if (needsBuiltinPatching(src)) {
+              vol.writeFileSync(fp, patchBuiltinImports(src));
+              converted++;
+            }
+            return;
+          }
+
+          // .mjs stays ESM even in non-ESM packages
+          if (fp.endsWith(".mjs")) {
+            if (needsBuiltinPatching(src)) {
+              vol.writeFileSync(fp, patchBuiltinImports(src));
+              converted++;
+            }
+            return;
+          }
+
+          if (requiresConversion(fp, src)) {
+            const out = await convertFile(src, fp);
+            vol.writeFileSync(fp, out);
+            converted++;
+          } else if (needsBuiltinPatching(src)) {
+            vol.writeFileSync(fp, patchBuiltinImports(src));
+            converted++;
+          }
+        } catch {
+        }
+      }),
+    );
+  }
+
+  return converted;
+}

package/src/packages/archive-extractor.ts

@@ -0,0 +1,248 @@
+// Archive Extractor — downloads .tgz from npm, decompresses, parses tar, writes to VFS.
+// Heavy work is offloaded to web workers when available.
+
+import pako from "pako";
+import { MemoryVolume } from "../memory-volume";
+import * as path from "../polyfills/path";
+import { offload, taskId, TaskPriority } from "../threading/offload";
+import type { ExtractResult } from "../threading/offload-types";
+import { base64ToBytes } from "../helpers/byte-encoding";
+import { precompileWasm } from "../helpers/wasm-cache";
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface ExtractionOptions {
+  // default 1 — strips npm's "package/" prefix
+  stripComponents?: number;
+  filter?: (entryPath: string) => boolean;
+  onProgress?: (msg: string) => void;
+}
+
+// ---------------------------------------------------------------------------
+// Internal tar structures
+// ---------------------------------------------------------------------------
+
+type EntryKind = "file" | "directory" | "link" | "other";
+
+interface ArchiveEntry {
+  filepath: string;
+  kind: EntryKind;
+  byteSize: number;
+  fileMode: number;
+  payload?: Uint8Array;
+  linkDestination?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Tar header helpers
+// ---------------------------------------------------------------------------
+
+function readNullTerminated(
+  buf: Uint8Array,
+  start: number,
+  len: number,
+): string {
+  const slice = buf.slice(start, start + len);
+  const zeroPos = slice.indexOf(0);
+  const trimmed = zeroPos >= 0 ? slice.slice(0, zeroPos) : slice;
+  return new TextDecoder().decode(trimmed);
+}
+
+function readOctalField(buf: Uint8Array, start: number, len: number): number {
+  const raw = readNullTerminated(buf, start, len).trim();
+  return parseInt(raw, 8) || 0;
+}
+
+function classifyTypeFlag(flag: string): EntryKind {
+  switch (flag) {
+    case "0":
+    case "\0":
+    case "":
+      return "file";
+    case "5":
+      return "directory";
+    case "1":
+    case "2":
+      return "link";
+    default:
+      return "other";
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Tar parser (generator)
+// ---------------------------------------------------------------------------
+
+export function* parseTarArchive(raw: Uint8Array): Generator<ArchiveEntry> {
+  const BLOCK = 512;
+  let cursor = 0;
+
+  while (cursor + BLOCK <= raw.length) {
+    const headerBlock = raw.slice(cursor, cursor + BLOCK);
+    cursor += BLOCK;
+
+    // two zero blocks = end of archive
+    const allZero = headerBlock.every((b) => b === 0);
+    if (allZero) break;
+
+    const nameField = readNullTerminated(headerBlock, 0, 100);
+    if (!nameField) continue;
+
+    const fileMode = readOctalField(headerBlock, 100, 8);
+    const byteSize = readOctalField(headerBlock, 124, 12);
+    const typeChar = String.fromCharCode(headerBlock[156]);
+    const linkField = readNullTerminated(headerBlock, 157, 100);
+    const prefixField = readNullTerminated(headerBlock, 345, 155);
+
+    const filepath = prefixField ? `${prefixField}/${nameField}` : nameField;
+    const kind = classifyTypeFlag(typeChar);
+
+    let payload: Uint8Array | undefined;
+    if (kind === "file") {
+      payload =
+        byteSize > 0 ? raw.slice(cursor, cursor + byteSize) : new Uint8Array(0);
+      if (byteSize > 0) {
+        cursor += Math.ceil(byteSize / BLOCK) * BLOCK;
+      }
+    }
+
+    yield {
+      filepath,
+      kind,
+      byteSize,
+      fileMode,
+      payload,
+      linkDestination: kind === "link" ? linkField : undefined,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Decompression
+// ---------------------------------------------------------------------------
+
+export function inflateGzip(compressed: ArrayBuffer | Uint8Array): Uint8Array {
+  const input =
+    compressed instanceof Uint8Array ? compressed : new Uint8Array(compressed);
+  return pako.inflate(input);
+}
+
+// ---------------------------------------------------------------------------
+// Extraction into MemoryVolume
+// ---------------------------------------------------------------------------
+
+export function extractArchive(
+  archiveBytes: ArrayBuffer | Uint8Array,
+  vol: MemoryVolume,
+  destDir: string,
+  opts: ExtractionOptions = {},
+): string[] {
+  const { stripComponents = 1, filter, onProgress } = opts;
+
+  onProgress?.("Inflating archive...");
+  const tarBytes = inflateGzip(archiveBytes);
+
+  const writtenPaths: string[] = [];
+
+  for (const entry of parseTarArchive(tarBytes)) {
+    if (entry.kind !== "file" && entry.kind !== "directory") continue;
+
+    let relative = entry.filepath;
+    if (stripComponents > 0) {
+      const segments = relative.split("/").filter(Boolean);
+      if (segments.length <= stripComponents) continue;
+      relative = segments.slice(stripComponents).join("/");
+    }
+
+    if (filter && !filter(relative)) continue;
+
+    const absolute = path.join(destDir, relative);
+
+    if (entry.kind === "directory") {
+      vol.mkdirSync(absolute, { recursive: true });
+    } else if (entry.kind === "file" && entry.payload) {
+      const parentDir = path.dirname(absolute);
+      vol.mkdirSync(parentDir, { recursive: true });
+      vol.writeFileSync(absolute, entry.payload);
+      // pre-compile so it's ready by the time code needs it
+      if (absolute.endsWith(".wasm")) {
+        precompileWasm(entry.payload);
+      }
+      writtenPaths.push(absolute);
+    }
+  }
+
+  onProgress?.(`Extracted ${writtenPaths.length} files`);
+  return writtenPaths;
+}
+
+// ---------------------------------------------------------------------------
+// High-level: download + extract in one step
+// ---------------------------------------------------------------------------
+
+// Offloads fetch + decompress + parse to a worker, then writes results to VFS on main thread
+export async function downloadAndExtract(
+  url: string,
+  vol: MemoryVolume,
+  destDir: string,
+  opts: ExtractionOptions = {},
+): Promise<string[]> {
+  opts.onProgress?.(`Fetching ${url}...`);
+
+  const result: ExtractResult = await offload({
+    type: "extract",
+    id: taskId(),
+    tarballUrl: url,
+    stripComponents: opts.stripComponents ?? 1,
+    priority: TaskPriority.NORMAL,
+  });
+
+  const writtenPaths: string[] = [];
+  for (const file of result.files) {
+    if (opts.filter && !opts.filter(file.path)) continue;
+
+    const absolute = path.join(destDir, file.path);
+    const parentDir = path.dirname(absolute);
+    vol.mkdirSync(parentDir, { recursive: true });
+
+    if (file.isBinary) {
+      vol.writeFileSync(absolute, base64ToBytes(file.data));
+    } else {
+      vol.writeFileSync(absolute, file.data);
+    }
+    writtenPaths.push(absolute);
+  }
+
+  opts.onProgress?.(`Extracted ${writtenPaths.length} files`);
+  return writtenPaths;
+}
+
+// Main-thread fallback when workers aren't available
+export async function downloadAndExtractDirect(
+  url: string,
+  vol: MemoryVolume,
+  destDir: string,
+  opts: ExtractionOptions = {},
+): Promise<string[]> {
+  opts.onProgress?.(`Fetching ${url}...`);
+
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(
+      `Archive download failed (HTTP ${response.status}): ${url}`,
+    );
+  }
+
+  const rawBytes = await response.arrayBuffer();
+  return extractArchive(rawBytes, vol, destDir, opts);
+}
+
+export default {
+  downloadAndExtract,
+  downloadAndExtractDirect,
+  parseTarArchive,
+  extractArchive,
+  inflateGzip,
+};