@alchemy.run/node-utils 2.0.0-beta.77 → 2.0.0-beta.78

package/src/register-oxc.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import * as NodeModule from "node:module";
 import {
   registerHooks,
   type LoadFnOutput,
@@ -7,10 +7,6 @@ import {
   type ResolveHookContext,
 } from "node:module";
 import { pathToFileURL } from "node:url";
-import type {
-  ImportLoader,
-  ImportLoaderRegistrationOptions,
-} from "./import-loader.ts";
 import {
   filePathOfUrl,
   isFileLikeSpecifier,
@@ -20,26 +16,69 @@ import {
 } from "./resolve-specifier.ts";
 import { SourceTransformer } from "./transform-source.ts";
 
-export type {
-  ImportLoader as RegisteredOxcImporter,
-  ImportLoaderRegistrationOptions as RegisterOxcOptions,
-  SourceTransform,
-  SourceTransformResult,
-  TransformContext,
-} from "./import-loader.ts";
+export interface OxcLoaderOptions {
+  /**
+   * Additional package export conditions used during module resolution.
+   * They are made available alongside Node's ambient conditions to both the
+   * TypeScript-aware resolver and Node's package exports resolver.
+   */
+  readonly conditions?: ReadonlyArray<string> | undefined;
+  /**
+   * Honour `tsconfig.json` discovered upward from each file: compiler
+   * options for the transform, `paths`/`baseUrl` aliases for resolution.
+   * @default true
+   */
+  readonly tsconfig?: boolean | undefined;
+  /** Controls which file URLs belong to the fresh import graph. */
+  readonly shouldInvalidate?:
+    | ((url: string, parentURL: string | undefined) => boolean)
+    | undefined;
+  /**
+   * Limits transformation to matching absolute file paths; everything else
+   * loads through Node untouched. Lets a published install transpile only
+   * the user's own TypeScript while alchemy and its dependencies run their
+   * built JavaScript.
+   */
+  readonly filter?: ((path: string) => boolean) | undefined;
+  /**
+   * On-disk cache of Oxc output shared by every process on the machine, so
+   * the CLI, its dev exec child and the local-provider sidecars transpile
+   * each source file once between them rather than once each. `false`
+   * disables it, a string names the directory.
+   * @default `$ALCHEMY_TRANSFORM_CACHE` (`0` disables), else a per-user
+   * directory under the OS temp directory
+   */
+  readonly cache?: boolean | string | undefined;
+}
+
+export interface RegisterOxcOptions extends OxcLoaderOptions {
+  /**
+   * Isolates one import graph in the runtime's module cache: every file
+   * URL the graph resolves carries this namespace as a query parameter, so
+   * the same files import again as fresh modules under a new namespace.
+   * This is how `alchemy dev` reloads the user's stack (see
+   * `watch-import.ts`); an un-namespaced registration is the process-wide
+   * TypeScript loader.
+   */
+  readonly namespace?: string | undefined;
+  /** Called once the runtime loads a file in this registration's graph. */
+  readonly onImport?: ((url: string) => void) | undefined;
+}
+
+export interface OxcLoader {
+  /**
+   * Imports a file under this registration's namespace. `specifier` is a
+   * file URL, an absolute path, or a path relative to `parentURL`.
+   */
+  import<T = unknown>(specifier: string, parentURL: string): Promise<T>;
+  unregister(): void;
+}
 
-const protocol = "alchemy-import:";
 const namespaceParameter = "alchemy-import-namespace";
 const globalRegistrationKey = Symbol.for(
   "@alchemy.run/node-utils/register-oxc",
 );
 
-interface ImportRequest {
-  readonly namespace?: string | undefined;
-  readonly parentURL: string;
-  readonly specifier: string;
-}
-
 type NextResolve = (
   specifier: string,
   context?: Partial<ResolveHookContext>,
@@ -63,45 +102,34 @@ const withNamespace = (url: string, namespace: string) => {
   return parsed.href;
 };
 
-const parseRequest = (specifier: string): ImportRequest | undefined => {
-  if (!specifier.startsWith(protocol)) return undefined;
-  return JSON.parse(decodeURIComponent(specifier.slice(protocol.length)));
-};
+/**
+ * Node's module compile cache (`module.enableCompileCache`) keeps V8 code
+ * cache for compiled modules — transformed TypeScript included, since it is
+ * keyed by the compiled source — but Node only persists it once after the
+ * entry module evaluated and again on a clean exit. Alchemy processes load
+ * most of their graph lazily after that point (commands, the user's stack,
+ * provider layers) and usually stop on a signal, so without an explicit
+ * flush that code never reaches the cache. Flush once module loading has
+ * gone quiet; a no-op when the cache is off or this Node predates it.
+ */
+const scheduleCompileCacheFlush = (() => {
+  let timer: NodeJS.Timeout | undefined;
+  return () => {
+    if (NodeModule.getCompileCacheDir?.() === undefined) return;
+    if (timer !== undefined) clearTimeout(timer);
+    timer = setTimeout(() => {
+      timer = undefined;
+      NodeModule.flushCompileCache?.();
+    }, 1000);
+    timer.unref();
+  };
+})();
 
 /** Specifiers Node owns outright: builtins, data URLs, remote schemes. */
 const isForeignSpecifier = (specifier: string) =>
   /^(?:node:|data:|[a-z][a-z\d+.-]*:\/\/)/i.test(specifier) &&
   !specifier.startsWith("file:");
 
-const notFoundCodes = new Set([
-  "ERR_MODULE_NOT_FOUND",
-  "MODULE_NOT_FOUND",
-  "ERR_UNSUPPORTED_DIR_IMPORT",
-  "ERR_PACKAGE_PATH_NOT_EXPORTED",
-]);
-
-const isNotFound = (error: unknown): error is Error & { url?: string } =>
-  error instanceof Error &&
-  notFoundCodes.has((error as { code?: string }).code ?? "");
-
-/**
- * The file Node could not find, from the error it raised. Node names the
- * resolved target (`url` on ESM errors, the message on CommonJS ones) —
- * for a package `exports` entry pointing at emitted JavaScript that was
- * never built, that is the `.js` path whose `.ts` source we can substitute.
- */
-const missingPathOf = (error: Error & { url?: string }) => {
-  if (error.url !== undefined) return filePathOfUrl(error.url);
-  const match = error.message.match(/^Cannot find module '([^']+)'/);
-  if (match === null) return undefined;
-  const [, target] = match;
-  if (target === undefined) return undefined;
-  if (target.startsWith("file:")) return filePathOfUrl(target);
-  return target.startsWith("/") || /^[a-zA-Z]:[\\/]/.test(target)
-    ? target
-    : undefined;
-};
-
 /** A `require()` reaching the hooks: Node's CommonJS resolver wants paths, not URLs. */
 const isRequireContext = (context: ResolveHookContext) =>
   context.conditions?.includes("require") === true &&
@@ -156,27 +184,7 @@ const resolveSpecifier = (
     }
   }
 
-  try {
-    return nextResolve(specifier, context);
-  } catch (error) {
-    if (!isNotFound(error)) throw error;
-    // A package `exports`/`main` target naming emitted JavaScript that only
-    // exists as TypeScript source (workspace packages in a checkout).
-    const missing = missingPathOf(error);
-    if (missing !== undefined && isFileLikeSpecifier(missing)) {
-      const candidate = resolver.resolveMissing(missing, conditions);
-      if (candidate !== undefined && candidate !== missing) {
-        const resolved = resolveWithCandidate(
-          candidate,
-          metadata,
-          context,
-          nextResolve,
-        );
-        if (resolved !== undefined) return resolved;
-      }
-    }
-    throw error;
-  }
+  return nextResolve(specifier, context);
 };
 
 /**
@@ -200,16 +208,14 @@ const withJsonAttribute = (url: string, context: LoadHookContext) => {
  * does. A namespaced registration also provides a scoped import whose
  * namespace propagates through the complete ESM graph.
  */
-export const registerOxc = (
-  options: ImportLoaderRegistrationOptions = {},
-): ImportLoader => {
+export const registerOxc = (options: RegisterOxcOptions = {}): OxcLoader => {
   // One global (un-namespaced) registration per process. Alchemy starts every
   // Node process with `--import` of a file that calls this, and in-process
   // callers (the dev exec child, tests) may call it again; a second copy of
   // the hooks would only re-run the resolve chain. The marker lives on
   // globalThis because a checkout can load this module twice (src/ and lib/).
   const globalRegistration = globalThis as typeof globalThis & {
-    [globalRegistrationKey]?: ImportLoader;
+    [globalRegistrationKey]?: OxcLoader;
   };
   if (options.namespace === undefined) {
     const existing = globalRegistration[globalRegistrationKey];
@@ -221,19 +227,20 @@ export const registerOxc = (
   });
   const shouldInvalidate = options.shouldInvalidate ?? (() => true);
 
-  // Transformed sources carry inline source maps; Node only applies them to
-  // stack traces once source-map support is on.
+  // Transformed sources reference their source maps (see transform-source);
+  // Node only reads and applies them to stack traces once source-map support
+  // is on.
   const sourceMapsWereEnabled = process.sourceMapsEnabled;
   process.setSourceMapsEnabled(true);
 
   const hooks = registerHooks({
     resolve(specifier, context, nextResolve) {
-      const request = parseRequest(specifier);
-      const inheritedNamespace = namespaceOf(context.parentURL);
+      // A graph's entry carries the namespace itself (see `import` below);
+      // everything it imports inherits it from the importing module's URL.
       const namespace =
         options.namespace === undefined
           ? undefined
-          : (request?.namespace ?? inheritedNamespace);
+          : (namespaceOf(specifier) ?? namespaceOf(context.parentURL));
 
       if (options.namespace !== undefined && namespace !== options.namespace) {
         return nextResolve(specifier, context);
@@ -248,14 +255,12 @@ export const registerOxc = (
                 ...new Set([...options.conditions, ...context.conditions]),
               ],
             };
-      const resolved = request
-        ? resolveSpecifier(
-            resolver,
-            request.specifier,
-            { ...resolutionContext, parentURL: request.parentURL },
-            nextResolve,
-          )
-        : resolveSpecifier(resolver, specifier, resolutionContext, nextResolve);
+      const resolved = resolveSpecifier(
+        resolver,
+        specifier,
+        resolutionContext,
+        nextResolve,
+      );
       if (
         namespace !== undefined &&
         resolved.url.startsWith("file:") &&
@@ -271,6 +276,7 @@ export const registerOxc = (
       return resolved;
     },
     load(url, context, nextLoad): LoadFnOutput {
+      scheduleCompileCacheFlush();
       const namespace = namespaceOf(url);
       if (options.namespace !== undefined && namespace !== options.namespace) {
         return nextLoad(url, context);
@@ -284,11 +290,7 @@ export const registerOxc = (
       if (options.filter !== undefined && !options.filter(filePath)) {
         return nextLoad(cleanUrl, withJsonAttribute(cleanUrl, context));
       }
-      const transformed = transformer.transform(
-        filePath,
-        cleanUrl,
-        context.format,
-      );
+      const transformed = transformer.transform(filePath, context.format);
       if (transformed === undefined) {
         return nextLoad(cleanUrl, withJsonAttribute(cleanUrl, context));
       }
@@ -296,17 +298,28 @@ export const registerOxc = (
     },
   });
 
-  const loader: ImportLoader = {
+  const loader: OxcLoader = {
     import<T>(specifier: string, parentURL: string) {
-      const request: ImportRequest = {
-        namespace: options.namespace,
-        parentURL: parentURL.startsWith("file:")
-          ? parentURL
-          : pathToFileURL(parentURL).href,
-        specifier,
-      };
+      if (!isFileLikeSpecifier(specifier)) {
+        throw new Error(
+          `Cannot import '${specifier}': expected a file URL or path.`,
+        );
+      }
+      const base = parentURL.startsWith("file:")
+        ? parentURL
+        : pathToFileURL(parentURL).href;
+      const url = specifier.startsWith("file:")
+        ? specifier
+        : new URL(
+            specifier.startsWith(".")
+              ? specifier
+              : pathToFileURL(specifier).href,
+            base,
+          ).href;
       return import(
-        `${protocol}${encodeURIComponent(JSON.stringify(request))}`
+        options.namespace === undefined
+          ? url
+          : withNamespace(url, options.namespace)
       ) as Promise<T>;
     },
     unregister() {
@@ -322,21 +335,3 @@ export const registerOxc = (
   }
   return loader;
 };
-
-/**
- * One-shot TypeScript import that leaves the rest of the runtime untouched:
- * a private namespace is registered for the call, so nothing is shared with
- * other imports of the same files. Mirrors tsx's `tsImport`.
- */
-export const tsImport = async <T = unknown>(
-  specifier: string,
-  parentURL: string,
-  options: Omit<ImportLoaderRegistrationOptions, "namespace"> = {},
-): Promise<T> => {
-  const loader = registerOxc({ ...options, namespace: randomUUID() });
-  try {
-    return await loader.import<T>(specifier, parentURL);
-  } finally {
-    await loader.unregister();
-  }
-};

package/src/resolve-specifier.ts

@@ -154,26 +154,4 @@ export class SpecifierResolver {
     }
     return result.path;
   }
-
-  /**
-   * Given a file path Node failed to find (typically an `exports`/`main`
-   * target that names emitted JavaScript that was never built), finds the
-   * TypeScript source it was emitted from via extension substitution.
-   */
-  resolveMissing(
-    missingPath: string,
-    conditions: ReadonlyArray<string>,
-  ): string | undefined {
-    const directory = path.dirname(missingPath);
-    const base = path.basename(missingPath);
-    try {
-      const result = this.#resolver(conditions, true).sync(
-        directory,
-        `./${base}`,
-      );
-      return result.path;
-    } catch {
-      return undefined;
-    }
-  }
 }

package/src/transform-cache.ts

@@ -0,0 +1,226 @@
+import { createHash } from "node:crypto";
+import {
+  existsSync,
+  mkdirSync,
+  promises as fs,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import rolldown from "rolldown/package.json" with { type: "json" };
+import self from "../package.json" with { type: "json" };
+
+/**
+ * Anything that changes Oxc's output for identical input invalidates every
+ * entry: the transformer itself (rolldown), this package's transform
+ * pipeline, and the on-disk entry layout.
+ */
+const CACHE_VERSION = ["1", self.version, rolldown.version].join("-");
+
+/** Entries untouched for this long are swept on the first disk access. */
+const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+export const TRANSFORM_CACHE_ENV = "ALCHEMY_TRANSFORM_CACHE";
+
+/**
+ * Per-user directory, like tsx's `tsx-<uid>`: `tmpdir()` is shared on
+ * multi-user machines, and entries are written with the owner's umask.
+ */
+const defaultDirectory = () => {
+  const user = process.geteuid?.() ?? os.userInfo().username;
+  return path.join(os.tmpdir(), `alchemy-oxc-${user}`);
+};
+
+/**
+ * The directory to use for the given option, or `undefined` when caching is
+ * off. `cache: false` and `ALCHEMY_TRANSFORM_CACHE=0` disable it; a string
+ * (option or env) names the directory; otherwise the per-user default.
+ */
+export const resolveCacheDirectory = (
+  option: boolean | string | undefined,
+): string | undefined => {
+  if (option === false) return undefined;
+  if (typeof option === "string") return option;
+  const env = process.env[TRANSFORM_CACHE_ENV];
+  if (env === "0" || env === "false") return undefined;
+  if (env !== undefined && env !== "") return env;
+  return defaultDirectory();
+};
+
+/** One transform's output as handed to {@link TransformCache.set}. */
+export interface TransformCacheEntry {
+  readonly format: "module" | "commonjs";
+  readonly code: string;
+  /** Source map JSON, or `undefined` when the transform produced none. */
+  readonly map: string | undefined;
+}
+
+/** A cache hit: the code, and where its source map lives on disk. */
+export interface CachedTransform {
+  readonly format: "module" | "commonjs";
+  readonly code: string;
+  /** Absolute path of the entry's `.map` file, present when it has one. */
+  readonly mapFile: string | undefined;
+}
+
+/**
+ * On-disk cache of Oxc transform output shared by every process on the
+ * machine — the `alchemy` CLI, its dev exec child, the local-provider
+ * sidecars and dev-server runners all load the same source files, and
+ * without this each of them transpiles the whole graph again.
+ *
+ * Modelled on tsx's file cache: entries are keyed by a hash of the source
+ * file's path, size and mtime, the transform options and the resolved
+ * tsconfig, so a change to any input is simply a different key; nothing is
+ * ever invalidated in place. Size plus nanosecond mtime stands in for the
+ * contents so a hit never reads the source.
+ *
+ * An entry is two files: `<key>.json` with the code, and `<key>.map` with
+ * the source map. The map stays on disk and is referenced from the module
+ * by path rather than inlined — an inline `data:` map is part of the
+ * script's source text, which V8 keeps for the process lifetime; for a
+ * graph the size of alchemy's that is hundreds of megabytes per process.
+ * Node reads the referenced file synchronously as it compiles the module,
+ * so the map is written (atomically) before `set` returns; the code entry
+ * is a fire-and-forget write, because a missing one is only a slower
+ * cache. Reads are synchronous (the loader hook is). Stale entries are
+ * swept by age once per process.
+ */
+export class TransformCache {
+  readonly #directory: string;
+  #ready = false;
+  #sequence = 0;
+
+  constructor(directory: string) {
+    this.#directory = directory;
+  }
+
+  /** The entry key for these transform inputs. */
+  key(parts: ReadonlyArray<string>): string {
+    const hash = createHash("sha1");
+    hash.update(CACHE_VERSION);
+    for (const part of parts) {
+      // Length-prefixed so adjacent parts cannot run into each other.
+      hash.update(`\0${part.length}\0`);
+      hash.update(part);
+    }
+    return hash.digest("hex");
+  }
+
+  /**
+   * The cached transform, or `undefined` on a miss. An entry whose map file
+   * has gone (swept, or never landed) is a miss too: the module would
+   * otherwise reference a map that is not there.
+   */
+  get(key: string): CachedTransform | undefined {
+    this.#prepare();
+    let raw: string;
+    try {
+      raw = readFileSync(this.#file(key, "json"), "utf8");
+    } catch {
+      return undefined;
+    }
+    let entry: { format?: unknown; code?: unknown; map?: unknown };
+    try {
+      entry = JSON.parse(raw);
+    } catch {
+      return undefined;
+    }
+    if (
+      (entry.format !== "module" && entry.format !== "commonjs") ||
+      typeof entry.code !== "string" ||
+      typeof entry.map !== "boolean"
+    ) {
+      return undefined;
+    }
+    const mapFile = this.#file(key, "map");
+    if (entry.map && !existsSync(mapFile)) return undefined;
+    return {
+      format: entry.format,
+      code: entry.code,
+      mapFile: entry.map ? mapFile : undefined,
+    };
+  }
+
+  /**
+   * Stores one transform. Returns the map file's path once it is on disk,
+   * or `undefined` when there is no map or it could not be written — the
+   * caller then falls back to inlining it.
+   */
+  set(key: string, entry: TransformCacheEntry): string | undefined {
+    this.#prepare();
+    let mapFile: string | undefined;
+    if (entry.map !== undefined) {
+      mapFile = this.#file(key, "map");
+      if (!this.#writeAtomically(mapFile, entry.map)) return undefined;
+    }
+    const file = this.#file(key, "json");
+    const temporary = this.#temporary(file);
+    // Best effort: a cache that cannot be written is only a slower cache.
+    fs.writeFile(
+      temporary,
+      JSON.stringify({
+        format: entry.format,
+        code: entry.code,
+        map: mapFile !== undefined,
+      }),
+    )
+      .then(() => fs.rename(temporary, file))
+      .catch(() => fs.unlink(temporary).catch(() => {}));
+    return mapFile;
+  }
+
+  #file(key: string, extension: "json" | "map") {
+    return path.join(this.#directory, `${key}.${extension}`);
+  }
+
+  #temporary(file: string) {
+    return `${file}.${process.pid}.${this.#sequence++}.tmp`;
+  }
+
+  /** Temp file plus rename: concurrent readers never see a partial file. */
+  #writeAtomically(file: string, content: string): boolean {
+    const temporary = this.#temporary(file);
+    try {
+      writeFileSync(temporary, content);
+      renameSync(temporary, file);
+      return true;
+    } catch {
+      try {
+        unlinkSync(temporary);
+      } catch {}
+      return false;
+    }
+  }
+
+  #prepare() {
+    if (this.#ready) return;
+    this.#ready = true;
+    try {
+      mkdirSync(this.#directory, { recursive: true });
+    } catch {
+      return;
+    }
+    // Off the hot path: the loader hook that got us here is synchronous.
+    setImmediate(() => {
+      this.#sweep().catch(() => {});
+    });
+  }
+
+  async #sweep() {
+    const cutoff = Date.now() - MAX_AGE_MS;
+    const names = await fs.readdir(this.#directory);
+    await Promise.all(
+      names.map(async (name) => {
+        const file = path.join(this.#directory, name);
+        try {
+          const { mtimeMs } = await fs.stat(file);
+          if (mtimeMs < cutoff) await fs.unlink(file);
+        } catch {}
+      }),
+    );
+  }
+}