@alchemy.run/node-utils 2.0.0-beta.76 → 2.0.0-beta.78

package/src/transform-cache.ts

@@ -0,0 +1,226 @@
+import { createHash } from "node:crypto";
+import {
+  existsSync,
+  mkdirSync,
+  promises as fs,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import rolldown from "rolldown/package.json" with { type: "json" };
+import self from "../package.json" with { type: "json" };
+
+/**
+ * Anything that changes Oxc's output for identical input invalidates every
+ * entry: the transformer itself (rolldown), this package's transform
+ * pipeline, and the on-disk entry layout.
+ */
+const CACHE_VERSION = ["1", self.version, rolldown.version].join("-");
+
+/** Entries untouched for this long are swept on the first disk access. */
+const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+export const TRANSFORM_CACHE_ENV = "ALCHEMY_TRANSFORM_CACHE";
+
+/**
+ * Per-user directory, like tsx's `tsx-<uid>`: `tmpdir()` is shared on
+ * multi-user machines, and entries are written with the owner's umask.
+ */
+const defaultDirectory = () => {
+  const user = process.geteuid?.() ?? os.userInfo().username;
+  return path.join(os.tmpdir(), `alchemy-oxc-${user}`);
+};
+
+/**
+ * The directory to use for the given option, or `undefined` when caching is
+ * off. `cache: false` and `ALCHEMY_TRANSFORM_CACHE=0` disable it; a string
+ * (option or env) names the directory; otherwise the per-user default.
+ */
+export const resolveCacheDirectory = (
+  option: boolean | string | undefined,
+): string | undefined => {
+  if (option === false) return undefined;
+  if (typeof option === "string") return option;
+  const env = process.env[TRANSFORM_CACHE_ENV];
+  if (env === "0" || env === "false") return undefined;
+  if (env !== undefined && env !== "") return env;
+  return defaultDirectory();
+};
+
+/** One transform's output as handed to {@link TransformCache.set}. */
+export interface TransformCacheEntry {
+  readonly format: "module" | "commonjs";
+  readonly code: string;
+  /** Source map JSON, or `undefined` when the transform produced none. */
+  readonly map: string | undefined;
+}
+
+/** A cache hit: the code, and where its source map lives on disk. */
+export interface CachedTransform {
+  readonly format: "module" | "commonjs";
+  readonly code: string;
+  /** Absolute path of the entry's `.map` file, present when it has one. */
+  readonly mapFile: string | undefined;
+}
+
+/**
+ * On-disk cache of Oxc transform output shared by every process on the
+ * machine — the `alchemy` CLI, its dev exec child, the local-provider
+ * sidecars and dev-server runners all load the same source files, and
+ * without this each of them transpiles the whole graph again.
+ *
+ * Modelled on tsx's file cache: entries are keyed by a hash of the source
+ * file's path, size and mtime, the transform options and the resolved
+ * tsconfig, so a change to any input is simply a different key; nothing is
+ * ever invalidated in place. Size plus nanosecond mtime stands in for the
+ * contents so a hit never reads the source.
+ *
+ * An entry is two files: `<key>.json` with the code, and `<key>.map` with
+ * the source map. The map stays on disk and is referenced from the module
+ * by path rather than inlined — an inline `data:` map is part of the
+ * script's source text, which V8 keeps for the process lifetime; for a
+ * graph the size of alchemy's that is hundreds of megabytes per process.
+ * Node reads the referenced file synchronously as it compiles the module,
+ * so the map is written (atomically) before `set` returns; the code entry
+ * is a fire-and-forget write, because a missing one is only a slower
+ * cache. Reads are synchronous (the loader hook is). Stale entries are
+ * swept by age once per process.
+ */
+export class TransformCache {
+  readonly #directory: string;
+  #ready = false;
+  #sequence = 0;
+
+  constructor(directory: string) {
+    this.#directory = directory;
+  }
+
+  /** The entry key for these transform inputs. */
+  key(parts: ReadonlyArray<string>): string {
+    const hash = createHash("sha1");
+    hash.update(CACHE_VERSION);
+    for (const part of parts) {
+      // Length-prefixed so adjacent parts cannot run into each other.
+      hash.update(`\0${part.length}\0`);
+      hash.update(part);
+    }
+    return hash.digest("hex");
+  }
+
+  /**
+   * The cached transform, or `undefined` on a miss. An entry whose map file
+   * has gone (swept, or never landed) is a miss too: the module would
+   * otherwise reference a map that is not there.
+   */
+  get(key: string): CachedTransform | undefined {
+    this.#prepare();
+    let raw: string;
+    try {
+      raw = readFileSync(this.#file(key, "json"), "utf8");
+    } catch {
+      return undefined;
+    }
+    let entry: { format?: unknown; code?: unknown; map?: unknown };
+    try {
+      entry = JSON.parse(raw);
+    } catch {
+      return undefined;
+    }
+    if (
+      (entry.format !== "module" && entry.format !== "commonjs") ||
+      typeof entry.code !== "string" ||
+      typeof entry.map !== "boolean"
+    ) {
+      return undefined;
+    }
+    const mapFile = this.#file(key, "map");
+    if (entry.map && !existsSync(mapFile)) return undefined;
+    return {
+      format: entry.format,
+      code: entry.code,
+      mapFile: entry.map ? mapFile : undefined,
+    };
+  }
+
+  /**
+   * Stores one transform. Returns the map file's path once it is on disk,
+   * or `undefined` when there is no map or it could not be written — the
+   * caller then falls back to inlining it.
+   */
+  set(key: string, entry: TransformCacheEntry): string | undefined {
+    this.#prepare();
+    let mapFile: string | undefined;
+    if (entry.map !== undefined) {
+      mapFile = this.#file(key, "map");
+      if (!this.#writeAtomically(mapFile, entry.map)) return undefined;
+    }
+    const file = this.#file(key, "json");
+    const temporary = this.#temporary(file);
+    // Best effort: a cache that cannot be written is only a slower cache.
+    fs.writeFile(
+      temporary,
+      JSON.stringify({
+        format: entry.format,
+        code: entry.code,
+        map: mapFile !== undefined,
+      }),
+    )
+      .then(() => fs.rename(temporary, file))
+      .catch(() => fs.unlink(temporary).catch(() => {}));
+    return mapFile;
+  }
+
+  #file(key: string, extension: "json" | "map") {
+    return path.join(this.#directory, `${key}.${extension}`);
+  }
+
+  #temporary(file: string) {
+    return `${file}.${process.pid}.${this.#sequence++}.tmp`;
+  }
+
+  /** Temp file plus rename: concurrent readers never see a partial file. */
+  #writeAtomically(file: string, content: string): boolean {
+    const temporary = this.#temporary(file);
+    try {
+      writeFileSync(temporary, content);
+      renameSync(temporary, file);
+      return true;
+    } catch {
+      try {
+        unlinkSync(temporary);
+      } catch {}
+      return false;
+    }
+  }
+
+  #prepare() {
+    if (this.#ready) return;
+    this.#ready = true;
+    try {
+      mkdirSync(this.#directory, { recursive: true });
+    } catch {
+      return;
+    }
+    // Off the hot path: the loader hook that got us here is synchronous.
+    setImmediate(() => {
+      this.#sweep().catch(() => {});
+    });
+  }
+
+  async #sweep() {
+    const cutoff = Date.now() - MAX_AGE_MS;
+    const names = await fs.readdir(this.#directory);
+    await Promise.all(
+      names.map(async (name) => {
+        const file = path.join(this.#directory, name);
+        try {
+          const { mtimeMs } = await fs.stat(file);
+          if (mtimeMs < cutoff) await fs.unlink(file);
+        } catch {}
+      }),
+    );
+  }
+}

package/src/transform-source.ts

@@ -0,0 +1,254 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { resolveTsconfig } from "rolldown/experimental";
+import {
+  parseSync,
+  transformSync,
+  TsconfigCache,
+  type TransformOptions,
+} from "rolldown/utils";
+import type { OxcLoaderOptions } from "./register-oxc.ts";
+import { resolveCacheDirectory, TransformCache } from "./transform-cache.ts";
+
+/** Extensions Oxc transpiles; everything else is JavaScript Node can run. */
+export const transformExtensions = new Set([
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".jsx",
+]);
+
+export type ModuleFormat = "module" | "commonjs";
+
+/**
+ * Module format from Node's `load` hook context. Node derives these from the
+ * extension and the nearest `package.json#type`; `*-typescript` variants are
+ * its TypeScript-aware spellings and mean the same thing.
+ */
+const nodeFormat = (
+  format: string | null | undefined,
+): ModuleFormat | undefined => {
+  switch (format) {
+    case "module":
+    case "module-typescript":
+      return "module";
+    case "commonjs":
+    case "commonjs-typescript":
+      return "commonjs";
+    default:
+      return undefined;
+  }
+};
+
+/** Fallback for older Nodes that pass no format: extension, then package type. */
+const inferFormat = (filePath: string): ModuleFormat => {
+  const extension = path.extname(filePath);
+  if (extension === ".mts" || extension === ".mjs") return "module";
+  if (extension === ".cts" || extension === ".cjs") return "commonjs";
+  let directory = path.dirname(filePath);
+  while (true) {
+    const packageJson = path.join(directory, "package.json");
+    if (existsSync(packageJson)) {
+      try {
+        return JSON.parse(readFileSync(packageJson, "utf8")).type === "module"
+          ? "module"
+          : "commonjs";
+      } catch {
+        return "commonjs";
+      }
+    }
+    const parent = path.dirname(directory);
+    if (parent === directory) return "commonjs";
+    directory = parent;
+  }
+};
+
+const language = (filePath: string): TransformOptions["lang"] => {
+  switch (path.extname(filePath)) {
+    case ".tsx":
+      return "tsx";
+    case ".ts":
+    case ".mts":
+    case ".cts":
+      return "ts";
+    case ".jsx":
+      return "jsx";
+    default:
+      return "js";
+  }
+};
+
+/**
+ * Inline map, for when there is no cache file to point at. Only the
+ * fallback: the base64 becomes part of the script source V8 retains, and
+ * for a non-ASCII source it is stored two bytes per character on top.
+ */
+const inlineSourceMapComment = (map: string | object) => {
+  const json = typeof map === "string" ? map : JSON.stringify(map);
+  return `\n//# sourceMappingURL=data:application/json;base64,${Buffer.from(json).toString("base64")}`;
+};
+
+/**
+ * Map by reference. Node's source-map support only understands `data:`
+ * URLs and scheme-less paths (it resolves the latter against the module
+ * URL and reads the file), so this is the file URL's path component —
+ * `/var/…/x.map` on POSIX, `/C:/…/x.map` on Windows — never a `file:` URL.
+ */
+const fileSourceMapComment = (mapFile: string) =>
+  `\n//# sourceMappingURL=${pathToFileURL(mapFile).pathname}`;
+
+/**
+ * The transform's map without `sourcesContent`. Every source is a file on
+ * this machine, named by the map's `sources`, so embedding its text only
+ * makes the map larger than the code it describes and every process that
+ * loads the module pay for it.
+ */
+const withoutSourcesContent = ({
+  sourcesContent: _,
+  ...map
+}: NonNullable<ReturnType<typeof transformSync>["map"]>) => map;
+
+export interface TransformedSource {
+  readonly format: ModuleFormat;
+  readonly source: string;
+}
+
+export class SourceTransformer {
+  readonly #options: OxcLoaderOptions;
+  readonly #tsconfigCache = new TsconfigCache();
+  readonly #cache: TransformCache | undefined;
+
+  constructor(options: OxcLoaderOptions) {
+    this.#options = options;
+    const directory = resolveCacheDirectory(options.cache);
+    this.#cache =
+      directory === undefined ? undefined : new TransformCache(directory);
+  }
+
+  /**
+   * Cache key for one transform, or `undefined` when the result must not be
+   * cached. Every input Oxc's output depends on is part of it: the file's
+   * path (source maps name it), its size and mtime standing in for its
+   * contents — a stat instead of a read plus a hash per module on the warm
+   * path — the effective transform options, Node's module format, and the
+   * tsconfig that would be discovered for the file, resolved through the
+   * same cache `transformSync` uses with the `extends` chain already
+   * merged, so editing any tsconfig in the chain is a new key.
+   */
+  #cacheKey(
+    filePath: string,
+    options: TransformOptions,
+    format: ModuleFormat,
+  ): string | undefined {
+    if (this.#cache === undefined) return undefined;
+    let stat: { size: bigint; mtimeNs: bigint };
+    try {
+      stat = statSync(filePath, { bigint: true });
+    } catch {
+      return undefined;
+    }
+    let tsconfig: unknown = null;
+    try {
+      if (options.tsconfig === true) {
+        tsconfig = resolveTsconfig(filePath, this.#tsconfigCache)?.tsconfig;
+      } else if (typeof options.tsconfig === "string") {
+        tsconfig = readFileSync(options.tsconfig, "utf8");
+      }
+    } catch {
+      // A broken tsconfig is the transform's error to report; don't cache.
+      return undefined;
+    }
+    return this.#cache.key([
+      filePath,
+      `${stat.size}:${stat.mtimeNs}`,
+      JSON.stringify(options),
+      JSON.stringify(tsconfig ?? null),
+      format,
+    ]);
+  }
+
+  /**
+   * Transpiles `filePath` for Node, or returns `undefined` when the file is
+   * JavaScript that needs no work. `format` is what Node's `load` hook was
+   * told; it decides `sourceType` and the format handed back.
+   */
+  transform(
+    filePath: string,
+    format: string | null | undefined,
+  ): TransformedSource | undefined {
+    const extension = path.extname(filePath);
+    if (!transformExtensions.has(extension)) return undefined;
+
+    let moduleFormat = nodeFormat(format) ?? inferFormat(filePath);
+    const lang = language(filePath);
+    const options: TransformOptions = {
+      tsconfig: this.#options.tsconfig ?? true,
+      sourcemap: true,
+      lang,
+    };
+    // The key is taken before the source is read so a hit costs one stat
+    // and one cache read, never the source itself.
+    const key = this.#cacheKey(filePath, options, moduleFormat);
+    const cached = key === undefined ? undefined : this.#cache?.get(key);
+    if (cached !== undefined) {
+      return {
+        format: cached.format,
+        source:
+          cached.mapFile === undefined
+            ? cached.code
+            : cached.code + fileSourceMapComment(cached.mapFile),
+      };
+    }
+    const source = readFileSync(filePath, "utf8");
+    // A `.ts` file in a CommonJS package that uses `import`/`export` runs
+    // as ESM — the same call Node's own module-syntax detection makes for
+    // `.js`. Explicit `.cts` stays CommonJS regardless.
+    if (
+      moduleFormat === "commonjs" &&
+      extension !== ".cts" &&
+      parseSync(filePath, source, { lang, sourceType: "unambiguous" }).module
+        .hasModuleSyntax
+    ) {
+      moduleFormat = "module";
+    }
+    const transformed = transformSync(
+      filePath,
+      source,
+      { ...options, sourceType: moduleFormat },
+      this.#tsconfigCache,
+    );
+    if (transformed.errors.length > 0) {
+      const [error] = transformed.errors;
+      throw error instanceof Error
+        ? error
+        : new SyntaxError(
+            `${filePath}: ${(error as { message?: string }).message ?? String(error)}`,
+          );
+    }
+    const map =
+      transformed.map === undefined
+        ? undefined
+        : JSON.stringify(withoutSourcesContent(transformed.map));
+    // The map ends up in the module exactly one way: on disk next to the
+    // cache entry and referenced by path, or (cache off) inlined.
+    const mapFile =
+      key === undefined || map === undefined
+        ? undefined
+        : this.#cache?.set(key, {
+            format: moduleFormat,
+            code: transformed.code,
+            map,
+          });
+    return {
+      format: moduleFormat,
+      source:
+        mapFile !== undefined
+          ? transformed.code + fileSourceMapComment(mapFile)
+          : map !== undefined
+            ? transformed.code + inlineSourceMapComment(map)
+            : transformed.code,
+    };
+  }
+}

package/src/watch-import-bun.ts

@@ -0,0 +1,99 @@
+import { realpathSync } from "node:fs";
+import path from "node:path";
+import {
+  DependencyWatcher,
+  type DependencyChangeListener,
+  type DependencyWatcherOptions,
+} from "./dependency-watcher.ts";
+
+export interface BunImportTrackerOptions extends DependencyWatcherOptions {
+  /**
+   * Directory whose modules belong to the tracked graph. Files outside it and
+   * anything under a `node_modules` directory load untouched.
+   */
+  readonly root: string;
+}
+
+const loaders: Record<string, "js" | "jsx" | "ts" | "tsx"> = {
+  ".js": "js",
+  ".mjs": "js",
+  ".cjs": "js",
+  ".jsx": "jsx",
+  ".ts": "ts",
+  ".mts": "ts",
+  ".cts": "ts",
+  ".tsx": "tsx",
+};
+
+const escapeRegExp = (value: string) =>
+  value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * Records every project-local module Bun loads after registration and watches
+ * those files for changes.
+ *
+ * Bun has no loader hooks that can evict or re-namespace an evaluated module,
+ * so unlike Node's {@link ImportWatcher} this cannot import a fresh
+ * generation in-process. A runtime `Bun.plugin` `onLoad` hook is used purely
+ * as a dependency probe: it hands the source back unchanged with the loader
+ * Bun would have picked itself. Callers react to a change by exiting so a
+ * supervisor can start a fresh process.
+ */
+export class BunImportTracker {
+  readonly #watcher: DependencyWatcher;
+  readonly #dependencies = new Set<string>();
+
+  constructor(options: BunImportTrackerOptions) {
+    if (process.versions.bun === undefined) {
+      throw new Error(
+        "BunImportTracker requires Bun; Node callers should use watchImport.",
+      );
+    }
+    this.#watcher = new DependencyWatcher(options);
+    // Bun reports real paths (`/private/tmp/...` for `/tmp/...` on macOS);
+    // match them against the root's real path too.
+    const root = realpathSync.native(path.resolve(options.root)) + path.sep;
+    const nodeModules = `node_modules${path.sep}`;
+    // `root` already ends in a separator, so the segment right after it has no
+    // leading one: a `.*${sep}node_modules${sep}` lookahead alone would miss
+    // `<root>node_modules/...` and intercept every dependency. Bun cannot
+    // return CommonJS source from `onLoad` (oven-sh/bun#19279), so a CJS
+    // dependency that reaches this probe loses its exports.
+    const filter = new RegExp(
+      `^${escapeRegExp(root)}(?!(?:.*${escapeRegExp(path.sep)})?${escapeRegExp(nodeModules)}).*\\.[cm]?[jt]sx?$`,
+    );
+    Bun.plugin({
+      name: "@alchemy.run/node-utils/watch-import-bun",
+      setup: (build) => {
+        build.onLoad({ filter }, async (args) => {
+          this.#dependencies.add(args.path);
+          this.#watcher.set(new Set(this.#dependencies));
+          return {
+            contents: await Bun.file(args.path).text(),
+            loader: loaders[path.extname(args.path)] ?? "js",
+          };
+        });
+      },
+    });
+  }
+
+  get dependencies(): ReadonlySet<string> {
+    return this.#watcher.dependencies;
+  }
+
+  subscribe(listener: DependencyChangeListener): () => void {
+    return this.#watcher.subscribe(listener);
+  }
+
+  /** Stops watching. The load hook stays registered but only echoes sources. */
+  close(): Promise<void> {
+    return this.#watcher.close();
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.close();
+  }
+}
+
+export const trackBunImports = (options: BunImportTrackerOptions) =>
+  new BunImportTracker(options);

package/src/watch-import.ts

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import {
+  DependencyWatcher,
+  type DependencyChangeListener,
+  type DependencyWatcherOptions,
+} from "./dependency-watcher.ts";
+import type { OxcLoader, OxcLoaderOptions } from "./register-oxc.ts";
+
+export interface ImportGeneration<T> {
+  readonly value: T;
+  readonly namespace: string;
+  readonly dependencies: ReadonlySet<string>;
+}
+
+export interface ImportWatcherOptions
+  extends OxcLoaderOptions, DependencyWatcherOptions {
+  readonly parentURL: string;
+}
+
+/**
+ * Imports fresh Node module generations and watches the exact files loaded by
+ * the current generation. Bun callers should use `BunImportTracker` from
+ * `./watch-import-bun.ts`: Bun cannot evict evaluated modules, so a change
+ * there restarts the process instead of importing a new generation.
+ */
+export class ImportWatcher<T = unknown> {
+  readonly #specifier: string;
+  readonly #options: ImportWatcherOptions;
+  readonly #watcher: DependencyWatcher;
+  #registration: OxcLoader | undefined;
+  #dependencies = new Set<string>();
+  #closed = false;
+
+  constructor(specifier: string, options: ImportWatcherOptions) {
+    this.#specifier = specifier;
+    this.#options = options;
+    this.#watcher = new DependencyWatcher(options);
+  }
+
+  get dependencies(): ReadonlySet<string> {
+    return this.#dependencies;
+  }
+
+  subscribe(listener: DependencyChangeListener): () => void {
+    return this.#watcher.subscribe(listener);
+  }
+
+  async import(): Promise<ImportGeneration<T>> {
+    if (this.#closed) throw new Error("ImportWatcher is closed");
+    const namespace = randomUUID();
+    const dependencies = new Set<string>();
+    const {
+      debounceMs: _,
+      parentURL,
+      watch: _watch,
+      ...registerOptions
+    } = this.#options;
+    // Loaded here, not at module scope: the exec child imports this file on
+    // both runtimes, and the loader's Node hooks do not exist under Bun.
+    const { registerOxc } = await import("./register-oxc.ts");
+    const registration = registerOxc({
+      ...registerOptions,
+      namespace,
+      onImport: (url) => {
+        if (!url.startsWith("file:")) return;
+        dependencies.add(fileURLToPath(url));
+        // A lazy import evaluated after this generation became current
+        // extends the watched set immediately.
+        if (this.#dependencies === dependencies)
+          this.#watcher.set(dependencies);
+      },
+    });
+    try {
+      const value = await registration.import<T>(this.#specifier, parentURL);
+      await this.#registration?.unregister();
+      this.#registration = registration;
+      this.#dependencies = dependencies;
+      this.#watcher.set(dependencies);
+      return { value, namespace, dependencies };
+    } catch (error) {
+      await registration.unregister();
+      // Keep watching everything the failed import touched so the next save
+      // of any of those files retries.
+      this.#dependencies = new Set([...this.#dependencies, ...dependencies]);
+      this.#watcher.set(this.#dependencies);
+      throw error;
+    }
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) return;
+    this.#closed = true;
+    await this.#registration?.unregister();
+    await this.#watcher.close();
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.close();
+  }
+}
+
+export const watchImport = <T = unknown>(
+  specifier: string,
+  options: ImportWatcherOptions,
+) => new ImportWatcher<T>(specifier, options);