@volter/tabnode 0.2.14-volter.88

package/src/npm/index.ts

@@ -0,0 +1,425 @@
+/**
+ * npm Package Manager
+ * Orchestrates package installation into the virtual file system
+ */
+
+import { VirtualFS } from '../virtual-fs';
+import { Registry, RegistryOptions } from './registry';
+import {
+  resolveDependencies,
+  resolveFromPackageJson,
+  ResolvedPackage,
+  ResolvedTree,
+  ResolveOptions,
+} from './resolver';
+import { downloadAndExtract, extractTarball, __browserRuntimeInstallPool } from './tarball';
+import * as path from '../shims/path';
+import { initTransformer, transformPackage, isTransformerReady } from '../transform';
+
+/**
+ * Whether the install rewrites the ES modules it unpacks into CommonJS.
+ *
+ * Off, whatever `transform` asked for: the pass was the largest share of the
+ * install's time, fourteen thousand files for a 700-package tree, and nothing
+ * needs it. The runtime's module loader transforms an ES module the first
+ * time it is required, with the result cached, and the browser-side module
+ * server bundles a package from its files with esbuild, which takes ES
+ * modules as they are. The install leaves the files as published.
+ *
+ * A live binding rather than a literal, so the transformer the loader itself
+ * uses stays part of the build instead of being folded away with the
+ * branches below.
+ */
+let installTransformEnabled = false;
+
+/** Turn the install's rewrite pass back on. Nothing in the engine does. */
+export function setInstallTransformEnabled(enabled: boolean): void {
+  installTransformEnabled = enabled;
+}
+
+/**
+ * Normalize a package.json bin field into a consistent Record<string, string>.
+ * Handles both string form ("bin": "cli.js") and object form ("bin": {"cmd": "cli.js"}).
+ */
+function normalizeBin(pkgName: string, bin?: Record<string, string> | string): Record<string, string> {
+  if (!bin) return {};
+  if (typeof bin === 'string') {
+    // String form uses the package name (without scope) as the command name
+    const cmdName = pkgName.includes('/') ? pkgName.split('/').pop()! : pkgName;
+    return { [cmdName]: bin };
+  }
+  return bin;
+}
+
+export interface InstallOptions {
+  registry?: string;
+  save?: boolean;
+  saveDev?: boolean;
+  includeDev?: boolean;
+  includeOptional?: boolean;
+  onProgress?: (message: string) => void;
+  /** Transform ESM packages to CJS after install (default: true) */
+  transform?: boolean;
+}
+
+export interface InstallResult {
+  installed: Map<string, ResolvedPackage>;
+  added: string[];
+}
+
+/**
+ * npm Package Manager for VirtualFS
+ */
+export class PackageManager {
+  private vfs: VirtualFS;
+  private registry: Registry;
+  private cwd: string;
+
+  constructor(vfs: VirtualFS, options: { cwd?: string } & RegistryOptions = {}) {
+    this.vfs = vfs;
+    this.registry = new Registry(options);
+    this.cwd = options.cwd || '/';
+  }
+
+  /**
+   * Install a package and its dependencies
+   */
+  async install(
+    packageSpec: string,
+    options: InstallOptions = {}
+  ): Promise<InstallResult> {
+    const { onProgress } = options;
+
+    // Parse package spec (name@version)
+    const { name, version } = parsePackageSpec(packageSpec);
+
+    onProgress?.(`Resolving ${name}@${version || 'latest'}...`);
+
+    // Resolve dependencies
+    const resolved = await resolveDependencies(name, version || 'latest', {
+      registry: this.registry,
+      includeDev: options.includeDev,
+      includeOptional: options.includeOptional,
+      onProgress,
+    });
+
+    // Install all resolved packages
+    const added = await this.installResolved(resolved, options);
+
+    // Update package.json if save option is set
+    if (options.save || options.saveDev) {
+      const pkgToAdd = resolved.get(name);
+      if (pkgToAdd) {
+        await this.updatePackageJson(
+          name,
+          `^${pkgToAdd.version}`,
+          options.saveDev || false
+        );
+      }
+    }
+
+    onProgress?.(`Installed ${resolved.size} packages`);
+
+    return { installed: resolved, added };
+  }
+
+  /**
+   * Install all dependencies from package.json
+   */
+  async installFromPackageJson(options: InstallOptions = {}): Promise<InstallResult> {
+    const { onProgress } = options;
+
+    const pkgJsonPath = path.join(this.cwd, 'package.json');
+
+    if (!this.vfs.existsSync(pkgJsonPath)) {
+      throw new Error('No package.json found');
+    }
+
+    // A workspace install runs member by member, and the page hands the
+    // engine the manifest each member is to be installed from: a pnpm or npm
+    // workspace rewrites a member's dependencies (a `workspace:*` link is not
+    // a registry range) and the file on disk is not what should be resolved.
+    // Where the page recorded no manifest for this directory, the file is.
+    const pkgJson = (this.vfs as unknown as Record<symbol, Map<string, unknown> | undefined>)[Symbol.for('@volter/browser-node/workspace-install-manifests')]?.get(this.cwd)
+      ?? JSON.parse(this.vfs.readFileSync(pkgJsonPath, 'utf8'));
+
+    onProgress?.('Resolving dependencies...');
+
+    // Resolve all dependencies
+    const resolved = await resolveFromPackageJson(pkgJson, {
+      registry: this.registry,
+      includeDev: options.includeDev,
+      includeOptional: options.includeOptional,
+      onProgress,
+    });
+
+    // Install all resolved packages
+    const added = await this.installResolved(resolved, options);
+
+    onProgress?.(`Installed ${resolved.size} packages`);
+
+    return { installed: resolved, added };
+  }
+
+  /**
+   * Install resolved packages to node_modules
+   */
+  private async installResolved(
+    resolved: ResolvedTree,
+    options: InstallOptions
+  ): Promise<string[]> {
+    const { onProgress } = options;
+    const added: string[] = [];
+
+    // Ensure node_modules exists
+    const nodeModulesPath = path.join(this.cwd, 'node_modules');
+    this.vfs.mkdirSync(nodeModulesPath, { recursive: true });
+
+    // Filter packages that need to be installed
+    const toInstall: Array<{ name: string; pkg: ResolvedPackage; pkgPath: string }> = [];
+
+    for (const [name, pkg] of resolved) {
+      const pkgPath = path.join(nodeModulesPath, name);
+
+      // Skip if already installed with same version
+      const existingPkgJson = path.join(pkgPath, 'package.json');
+      if (this.vfs.existsSync(existingPkgJson)) {
+        try {
+          const existing = JSON.parse(
+            this.vfs.readFileSync(existingPkgJson, 'utf8')
+          );
+          if (existing.version === pkg.version) {
+            onProgress?.(`Skipping ${name}@${pkg.version} (already installed)`);
+            continue;
+          }
+        } catch {
+          // Continue with installation if package.json is invalid
+        }
+      }
+
+      toInstall.push({ name, pkg, pkgPath });
+    }
+
+    // The versions a dependent gets under its own node_modules, from the
+    // resolver's nested walk: React Router's build found semver 6 where
+    // normalize-package-data wanted 7 while the tree stayed flat.
+    for (const [nestedKey, pkg] of resolved.__nested instanceof Map ? resolved.__nested : []) {
+      toInstall.push({ name: pkg.name, pkg, pkgPath: path.join(nodeModulesPath, nestedKey) });
+    }
+
+    // The install rewrote every ES module it unpacked into CommonJS with
+    // esbuild, fourteen thousand files for a 700-package tree, and that pass
+    // was the largest share of the install's time. Nothing needs it: the
+    // runtime's module loader transforms an ES module the first time it is
+    // required, with the result cached, and the browser-side module server
+    // bundles a package from its files with esbuild, which takes ES modules
+    // as they are. The install leaves the files as published.
+    const shouldTransform = installTransformEnabled;
+    if (shouldTransform && !isTransformerReady()) {
+      onProgress?.('Initializing ESM transformer...');
+      await initTransformer();
+    }
+
+    // Six packages at a time, each batch waiting for its slowest, left the
+    // network idle behind one large tarball. A pool keeps every lane busy:
+    // as many downloads in flight as the page allows connections, and the
+    // extraction they feed serialized under the store's lock.
+    const CONCURRENCY = 16;
+    onProgress?.(`Installing ${toInstall.length} packages...`);
+
+    await __browserRuntimeInstallPool(toInstall, CONCURRENCY, async ({ name, pkg, pkgPath }) => {
+          onProgress?.(`  Downloading ${name}@${pkg.version}...`);
+
+          // With a shared package store, a package the origin has seen is
+          // linked instead of downloaded: the hook the worker installs is
+          // asked first, and only a declined package is fetched.
+          const __installHook = (globalThis as Record<symbol, ((name: string, pkg: ResolvedPackage, pkgPath: string, vfs: VirtualFS) => Promise<boolean>) | undefined>)[Symbol.for("@volter/browser-runtime/package-install")];
+          if (!(__installHook && await __installHook(name, pkg, pkgPath, this.vfs))) {
+            await downloadAndExtract(pkg.tarballUrl, this.vfs, pkgPath, {
+              stripComponents: 1, // Strip "package/" prefix
+            });
+          }
+
+          // Transform ESM to CJS
+          if (shouldTransform) {
+            try {
+              const count = await transformPackage(this.vfs, pkgPath, onProgress);
+              if (count > 0) {
+                onProgress?.(`  Transformed ${count} files in ${name}`);
+              }
+            } catch (transformError) {
+              onProgress?.(`  Warning: Transform failed for ${name}: ${transformError}`);
+            }
+          }
+
+          // Create bin stubs in /node_modules/.bin/
+          try {
+            const pkgJsonPath = path.join(pkgPath, 'package.json');
+            if (this.vfs.existsSync(pkgJsonPath)) {
+              const pkgJson = JSON.parse(this.vfs.readFileSync(pkgJsonPath, 'utf8'));
+              const binEntries = normalizeBin(name, pkgJson.bin);
+              // A nested package's stubs belong to the node_modules it sits in.
+              const binDir = path.join(pkgPath.slice(0, pkgPath.lastIndexOf("/node_modules/") + "/node_modules".length), ".bin");
+              for (const [cmdName, entryPath] of Object.entries(binEntries)) {
+                this.vfs.mkdirSync(binDir, { recursive: true });
+                const targetPath = path.join(pkgPath, entryPath);
+                this.vfs.writeFileSync(
+                  path.join(binDir, cmdName),
+                  `node "${targetPath}" "$@"\n`
+                );
+              }
+            }
+          } catch {
+            // Non-critical — skip if bin stub creation fails
+          }
+
+          added.push(name);
+    });
+
+    // Create .package-lock.json for tracking
+    await this.writeLockfile(resolved);
+
+    return added;
+  }
+
+  /**
+   * Write lockfile with resolved versions
+   */
+  private async writeLockfile(
+    resolved: Map<string, ResolvedPackage>
+  ): Promise<void> {
+    const lockfile: Record<string, { version: string; resolved: string }> = {};
+
+    for (const [name, pkg] of resolved) {
+      lockfile[name] = {
+        version: pkg.version,
+        resolved: pkg.tarballUrl,
+      };
+    }
+
+    const lockfilePath = path.join(this.cwd, 'node_modules', '.package-lock.json');
+    this.vfs.writeFileSync(lockfilePath, JSON.stringify(lockfile, null, 2));
+  }
+
+  /**
+   * Update package.json with new dependency
+   */
+  private async updatePackageJson(
+    packageName: string,
+    version: string,
+    isDev: boolean
+  ): Promise<void> {
+    const pkgJsonPath = path.join(this.cwd, 'package.json');
+
+    let pkgJson: Record<string, unknown> = {};
+
+    if (this.vfs.existsSync(pkgJsonPath)) {
+      pkgJson = JSON.parse(this.vfs.readFileSync(pkgJsonPath, 'utf8'));
+    }
+
+    const field = isDev ? 'devDependencies' : 'dependencies';
+
+    if (!pkgJson[field]) {
+      pkgJson[field] = {};
+    }
+
+    (pkgJson[field] as Record<string, string>)[packageName] = version;
+
+    this.vfs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2));
+  }
+
+  /**
+   * List installed packages
+   */
+  list(): Record<string, string> {
+    const nodeModulesPath = path.join(this.cwd, 'node_modules');
+
+    if (!this.vfs.existsSync(nodeModulesPath)) {
+      return {};
+    }
+
+    const packages: Record<string, string> = {};
+    const entries = this.vfs.readdirSync(nodeModulesPath);
+
+    for (const entry of entries) {
+      // Skip hidden files and non-package entries
+      if (entry.startsWith('.')) continue;
+
+      // Handle scoped packages (@org/pkg)
+      if (entry.startsWith('@')) {
+        const scopePath = path.join(nodeModulesPath, entry);
+        const scopedPkgs = this.vfs.readdirSync(scopePath);
+
+        for (const scopedPkg of scopedPkgs) {
+          const pkgJsonPath = path.join(scopePath, scopedPkg, 'package.json');
+          if (this.vfs.existsSync(pkgJsonPath)) {
+            const pkgJson = JSON.parse(this.vfs.readFileSync(pkgJsonPath, 'utf8'));
+            packages[`${entry}/${scopedPkg}`] = pkgJson.version;
+          }
+        }
+      } else {
+        const pkgJsonPath = path.join(nodeModulesPath, entry, 'package.json');
+        if (this.vfs.existsSync(pkgJsonPath)) {
+          const pkgJson = JSON.parse(this.vfs.readFileSync(pkgJsonPath, 'utf8'));
+          packages[entry] = pkgJson.version;
+        }
+      }
+    }
+
+    return packages;
+  }
+}
+
+/**
+ * Parse a package specifier into name and version
+ * Examples: "express", "express@4.18.2", "@types/node@18"
+ */
+function parsePackageSpec(spec: string): { name: string; version?: string } {
+  // Handle scoped packages
+  if (spec.startsWith('@')) {
+    const slashIndex = spec.indexOf('/');
+    if (slashIndex === -1) {
+      throw new Error(`Invalid package spec: ${spec}`);
+    }
+
+    const afterSlash = spec.slice(slashIndex + 1);
+    const atIndex = afterSlash.indexOf('@');
+
+    if (atIndex === -1) {
+      return { name: spec };
+    }
+
+    return {
+      name: spec.slice(0, slashIndex + 1 + atIndex),
+      version: afterSlash.slice(atIndex + 1),
+    };
+  }
+
+  // Regular packages
+  const atIndex = spec.indexOf('@');
+  if (atIndex === -1) {
+    return { name: spec };
+  }
+
+  return {
+    name: spec.slice(0, atIndex),
+    version: spec.slice(atIndex + 1),
+  };
+}
+
+// Convenience function for quick installs
+export async function install(
+  packageSpec: string,
+  vfs: VirtualFS,
+  options?: InstallOptions
+): Promise<InstallResult> {
+  const pm = new PackageManager(vfs);
+  return pm.install(packageSpec, options);
+}
+
+// Re-export types and modules
+export { Registry } from './registry';
+export type { RegistryOptions, PackageVersion, PackageManifest } from './registry';
+export type { ResolvedPackage, ResolveOptions } from './resolver';
+export type { ExtractOptions } from './tarball';
+export { parsePackageSpec };

package/src/npm/registry.ts

@@ -0,0 +1,235 @@
+/**
+ * npm Registry Client
+ * Fetches package metadata from npm registry
+ */
+
+export interface PackageVersion {
+  name: string;
+  version: string;
+  dependencies?: Record<string, string>;
+  devDependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, { optional?: boolean }>;
+  optionalDependencies?: Record<string, string>;
+  dist: {
+    tarball: string;
+    shasum: string;
+    integrity?: string;
+  };
+  main?: string;
+  module?: string;
+  exports?: Record<string, unknown>;
+  bin?: Record<string, string> | string;
+}
+
+export interface PackageManifest {
+  name: string;
+  'dist-tags': {
+    latest: string;
+    [tag: string]: string;
+  };
+  versions: Record<string, PackageVersion>;
+  time?: Record<string, string>;
+}
+
+export interface RegistryOptions {
+  registry?: string;
+  /** An `@scope`, or a whole unscoped name, to the registry that answers for it. */
+  registryScopes?: Record<string, string>;
+  cache?: Map<string, PackageManifest>;
+}
+
+declare global {
+  /**
+   * The version a lockfile recorded for a name, as the page read it: enough
+   * to stand in for the registry's document without fetching the packument.
+   */
+  // eslint-disable-next-line no-var
+  var __browserRuntimeLockfileManifests: Map<string, LockfileVersionDoc> | undefined;
+  /**
+   * An `@scope`, or a whole unscoped name, to the registry that answers for
+   * it. The engine builds its package manager itself, with no options of
+   * ours, so a page's scopes arrive the way the lockfile's records do.
+   */
+  // eslint-disable-next-line no-var
+  var __browserRuntimeRegistryScopes: Record<string, string> | undefined;
+}
+
+/** One version's record in a lockfile, as the page hands it over. */
+export interface LockfileVersionDoc {
+  version: string;
+  dependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, { optional?: boolean }>;
+  optionalDependencies?: Record<string, string>;
+}
+
+const DEFAULT_REGISTRY = 'https://registry.npmjs.org';
+
+export class Registry {
+  // Read by the resolver when it forms a tarball URL for a nested version.
+  registryUrl: string;
+  registryScopes?: Record<string, string>;
+  private cache: Map<string, PackageManifest>;
+
+  constructor(options: RegistryOptions = {}) {
+    this.registryUrl = options.registry || DEFAULT_REGISTRY;
+    this.registryScopes = options.registryScopes;
+    this.cache = options.cache || new Map();
+  }
+
+  /**
+   * Which registry answers for a name: the one its scope is served from,
+   * or the one its whole name is, and otherwise the default.
+   *
+   * A project may carry packages of its own that no public registry has: a
+   * private SDK, a twin, a package built in the same workspace but never
+   * published. Asking one registry for every name meant npm answered 404 for
+   * the scope and the install failed with nothing the project could do short
+   * of publishing.
+   */
+  registryFor(packageName: string): string {
+    const scopes = this.registryScopes ?? globalThis.__browserRuntimeRegistryScopes;
+    if (scopes && typeof packageName === "string") {
+      const scope = packageName.startsWith("@") && packageName.includes("/") ? packageName.slice(0, packageName.indexOf("/")) : packageName;
+      const url = scopes[scope] ?? scopes[packageName];
+      if (typeof url === "string" && url !== "") return url.endsWith("/") ? url.slice(0, -1) : url;
+    }
+    return this.registryUrl;
+  }
+
+  /**
+   * Fetch package manifest (all versions metadata)
+   */
+  async getPackageManifest(packageName: string): Promise<PackageManifest> {
+    // Check cache first
+    if (this.cache.has(packageName)) {
+      return this.cache.get(packageName)!;
+    }
+
+    // With a lockfile in hand the resolver already knows every version, so the
+    // whole packument, every version a package ever published, is the wrong
+    // request: it is the largest download of an install and the slowest part
+    // of a cold one. A pinned name asks for that one version instead, and the
+    // answer is shaped like a one-version packument so the rest of the
+    // resolver is unchanged.
+    const __docs = globalThis.__browserRuntimeLockfileManifests;
+    const __doc = __docs instanceof Map ? __docs.get(packageName) : void 0;
+    if (__doc) {
+      // The lockfile's record of the version stands in for the registry's
+      // document: the tarball is where a registry keeps one, and what the
+      // version depends on is what the lockfile wrote down.
+      const __tarball = `${this.registryFor(packageName)}/${encodePackageName(packageName)}/-/${packageName.slice(packageName.lastIndexOf("/") + 1)}-${__doc.version}.tgz`;
+      const __manifest = { "dist-tags": { latest: __doc.version }, versions: { [__doc.version]: { name: packageName, version: __doc.version, dist: { tarball: __tarball }, dependencies: __doc.dependencies, ...(__doc.peerDependencies ? { peerDependencies: __doc.peerDependencies, peerDependenciesMeta: __doc.peerDependenciesMeta || {} } : {}), ...(__doc.optionalDependencies ? { optionalDependencies: __doc.optionalDependencies } : {}) } } } as unknown as PackageManifest;
+      this.cache.set(packageName, __manifest);
+      return __manifest;
+    }
+    const __pins = globalThis.__browserRuntimeLockfilePins;
+    const __pin = __pins instanceof Map ? __pins.get(packageName) : void 0;
+    if (typeof __pin === "string") {
+      const pinnedUrl = `${this.registryFor(packageName)}/${encodePackageName(packageName)}/${__pin}`;
+      const pinnedResponse = await fetch(pinnedUrl, { headers: { Accept: "application/json" } });
+      if (pinnedResponse.ok) {
+        const versionData = await pinnedResponse.json() as PackageVersion;
+        if (versionData && versionData.dist && versionData.dist.tarball) {
+          const pinnedManifest = { "dist-tags": { latest: __pin }, versions: { [__pin]: versionData } } as unknown as PackageManifest;
+          this.cache.set(packageName, pinnedManifest);
+          return pinnedManifest;
+        }
+      }
+    }
+
+    const url = `${this.registryFor(packageName)}/${encodePackageName(packageName)}`;
+
+    const response = await fetch(url, {
+      headers: {
+        Accept: 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8',
+      },
+    });
+
+    if (!response.ok) {
+      if (response.status === 404) {
+        throw new Error(`Package not found: ${packageName}`);
+      }
+      throw new Error(`Failed to fetch package ${packageName}: ${response.status}`);
+    }
+
+    const manifest = (await response.json()) as PackageManifest;
+
+    // Cache the result
+    this.cache.set(packageName, manifest);
+
+    return manifest;
+  }
+
+  /**
+   * Get specific version metadata
+   */
+  async getPackageVersion(
+    packageName: string,
+    version: string
+  ): Promise<PackageVersion> {
+    const manifest = await this.getPackageManifest(packageName);
+
+    // Handle dist-tags (like "latest", "next", etc.)
+    if (manifest['dist-tags'][version]) {
+      version = manifest['dist-tags'][version];
+    }
+
+    const versionData = manifest.versions[version];
+    if (!versionData) {
+      throw new Error(`Version ${version} not found for package ${packageName}`);
+    }
+
+    return versionData;
+  }
+
+  /**
+   * Get latest version number
+   */
+  async getLatestVersion(packageName: string): Promise<string> {
+    const manifest = await this.getPackageManifest(packageName);
+    return manifest['dist-tags'].latest;
+  }
+
+  /**
+   * Get all available versions
+   */
+  async getVersions(packageName: string): Promise<string[]> {
+    const manifest = await this.getPackageManifest(packageName);
+    return Object.keys(manifest.versions);
+  }
+
+  /**
+   * Download tarball as ArrayBuffer
+   */
+  async downloadTarball(tarballUrl: string): Promise<ArrayBuffer> {
+    const response = await fetch(tarballUrl);
+
+    if (!response.ok) {
+      throw new Error(`Failed to download tarball: ${response.status}`);
+    }
+
+    return response.arrayBuffer();
+  }
+
+  /**
+   * Clear the cache
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+}
+
+/**
+ * Encode scoped package names for URL
+ * @scoped/package -> @scoped%2fpackage
+ */
+export function encodePackageName(name: string): string {
+  return name.replace('/', '%2f');
+}
+
+// Default registry instance
+export const registry = new Registry();
+
+export default Registry;