@uniflowed/host 0.0.0-alpha.2 → 0.0.0-alpha.5

package/internal/node-hooks.js

@@ -11,46 +11,36 @@
 // left to Node.
 //
 // Transforms are cached on disk under `.uf/cache/transform/` keyed by a hash
-// of the source, so a second run of the same file is a read rather than a
-// round trip. The cache is content-addressed: an edited file hashes
-// differently, so there is no invalidation to get wrong.
+// of the source *and* of the `uf` that compiled it, so a second run of the
+// same file is a read rather than a round trip.
+//
+// Both halves are load-bearing. The key was the source alone at first, on the
+// reasoning that a content-addressed cache has no invalidation to get wrong —
+// which quietly assumed the compiler was a constant. It is not: edit
+// `crates/uf_transform` or `crates/uf_stylex`, rebuild, run `uf test`, and
+// every module whose *source* had not changed came back as the previous
+// binary had compiled it. The suite then passed, or failed, for the previous
+// build's reasons, and the only symptom was an answer that made no sense.
+// `rm -rf .uf/cache/transform` was the cure, and finding that out cost a
+// debugging session while `@uniflowed/stylex`'s preset was being written.
 
 import { createHash } from "node:crypto";
-import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
-import { isFlowModule, transformFlow } from "../transform.js";
+import { isFlowModule, sharedService, transformFlow, ufBinaryIdentity } from "../transform.js";
+import { writeAtomically } from "../write-atomically.js";
 
 /**
- * Write `contents` to `target` so a concurrent reader never sees half of it.
- *
- * `uf test` runs one of these processes per core and they all import the same
- * few modules at once, so two writers and a reader meet on the same cache
- * entry constantly. `writeFileSync` is not atomic — a reader can observe a
- * truncated file and report a module that "does not provide an export" — so
- * the content goes to a private temporary name first and is then renamed,
- * which is atomic within a filesystem.
+ * Bumped whenever *this file's* framing of the output changes, to retire old
+ * entries.
  *
- * A failure here is not a failure: a read-only checkout still runs, just
- * without the cache.
+ * Not the compiler's version, which is `ufBinaryIdentity` and which nobody
+ * has to remember. What is left for this to cover is what the loader adds
+ * around a transform — the appended source map, the module format it forces —
+ * and that is all it should ever be bumped for.
  */
-function writeAtomically(target, contents) {
-  const temporary = `${target}.${process.pid}.${Math.random().toString(36).slice(2)}`;
-  try {
-    mkdirSync(cacheDirectory, { recursive: true });
-    writeFileSync(temporary, contents);
-    renameSync(temporary, target);
-  } catch {
-    try {
-      unlinkSync(temporary);
-    } catch {
-      // Nothing to clean up.
-    }
-  }
-}
-
-/** Bumped whenever the transform's output shape changes, to retire old entries. */
 const CACHE_VERSION = "2";
 
 let cacheDirectory = null;
@@ -82,15 +72,56 @@ export async function load(url, context, nextLoad) {
   return { format: "module", source: code, shortCircuit: true };
 }
 
-async function cachedTransform(source, filename) {
+/**
+ * The file this module's compiled form belongs in under `identity`, or `null`
+ * when it must not be cached at all.
+ *
+ * `null` when there is no cache directory, and — the case worth spelling out —
+ * when the caller has no identity to give: nothing is read and nothing is
+ * written. Hashing the rest anyway would give every build of `uf` one key
+ * again, and writing under it would leave an entry for the next run to trust.
+ * A host that cannot name its compiler compiles everything, every time, which
+ * is slower and is never wrong.
+ */
+function cacheEntryFor(identity, source, filename) {
+  if (cacheDirectory == null || identity == null) return null;
   const key = createHash("sha256")
     .update(CACHE_VERSION)
     .update("\0")
+    .update(identity)
+    .update("\0")
     .update(filename)
     .update("\0")
     .update(source)
     .digest("hex");
-  const entry = cacheDirectory ? path.join(cacheDirectory, `${key}.mjs`) : null;
+  return path.join(cacheDirectory, `${key}.mjs`);
+}
+
+/**
+ * The compiled form of one module, from disk if some build already produced
+ * it and from `uf` otherwise.
+ *
+ * The two keys are computed from two different identities on purpose.
+ *
+ * The **read** is keyed by the binary as it is *now*, stat'd per module rather
+ * than once when the hooks were installed. A rebuild between installing them
+ * and loading the first Flow module would otherwise serve the old build's
+ * output while the new one is what would run — the same staleness this key
+ * exists to remove, with a smaller window. A stat is a microsecond and a
+ * fully warm run still spawns nothing, which is the property that decided the
+ * key's shape in the first place.
+ *
+ * The **write** is keyed by the binary the compiler process is actually
+ * executing, which `sharedService` read before it spawned and which cannot
+ * change afterwards. Keying the write by the file's current state would file
+ * this build's output under the next build's name if the rebuild landed while
+ * the module was being compiled — the same lie pointing the other way.
+ *
+ * They are usually the same string. When they are not, a rebuild happened
+ * during this run, and each half is right about its own half.
+ */
+async function cachedTransform(source, filename) {
+  const entry = cacheEntryFor(ufBinaryIdentity(), source, filename);
 
   if (entry) {
     try {
@@ -106,8 +137,11 @@ async function cachedTransform(source, filename) {
     ? `${out.code}\n//# sourceMappingURL=data:application/json;base64,${Buffer.from(out.map).toString("base64")}\n`
     : out.code;
 
-  if (entry) {
-    writeAtomically(entry, output);
+  const written = cacheEntryFor(sharedService(root).identity, source, filename);
+  if (written) {
+    // Tolerant: a cache that cannot be written is a slower run, not a
+    // failed one — a read-only checkout still works.
+    writeAtomically(written, output, { tolerant: true });
   }
   return output;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniflowed/host",
-  "version": "0.0.0-alpha.2",
+  "version": "0.0.0-alpha.5",
   "description": "Running Flow on a Capability JS Host, with no bundler in the way.",
   "type": "module",
   "license": "MIT",
@@ -11,10 +11,11 @@
     "directory": "packages/host"
   },
   "exports": {
-    "./register": "./register.js",
     "./bun-preload": "./bun-preload.js",
+    "./internal/node-hooks.js": "./internal/node-hooks.js",
+    "./register": "./register.js",
     "./transform": "./transform.js",
-    "./internal/node-hooks.js": "./internal/node-hooks.js"
+    "./write-atomically": "./write-atomically.js"
   },
   "files": [
     "register.js",

package/transform.js

@@ -14,6 +14,8 @@
 // all produce the same module from the same source.
 
 import { spawn } from "node:child_process";
+import { accessSync, constants, statSync } from "node:fs";
+import path from "node:path";
 import { createInterface } from "node:readline";
 
 /** File extensions uf treats as Flow source. */
@@ -60,6 +62,91 @@ export function ufBinary() {
   return process.env.UF_BINARY ?? "uf";
 }
 
+/**
+ * Which *build* of `uf` a host will transform through, or `null` when that
+ * cannot be established.
+ *
+ * `ufBinary()` names the compiler; this identifies it. Anything kept across
+ * runs needs the second, because the first does not change when the compiler
+ * does: `crates/uf_transform` is edited, `cargo build` writes a new binary
+ * over the old one, and every answer already on disk is now wrong while the
+ * name that produced them is unchanged. A version string is the same promise
+ * one step removed — every build between two releases shares one.
+ *
+ * So: the size and modification time of the file that will be executed. They
+ * move together on every rebuild, they are one `stat` away, and — this is the
+ * part that decided it — reading them does not require starting `uf`. A run
+ * that finds everything already compiled must not have to spawn the compiler
+ * to learn that it does not need it, which is what asking the running
+ * `uf transform` to introduce itself would have cost.
+ *
+ * The same test is applied to a path as to a bare name: a regular file with
+ * the execute bit. Size and mtime do not move when a binary loses that bit, so
+ * without the test a chmod produced the same identity as before, a warm cache
+ * went on serving, and a cold one failed to start `uf` — the answer depending
+ * on how warm the cache was, which is the class of bug this key exists to
+ * remove.
+ *
+ * `null` means the question could not be answered. It is not an invitation to
+ * hash the rest anyway: a key that leaves the compiler out is one key for
+ * every build of it, which is the whole defect.
+ *
+ * @param {string} [command] the binary; `ufBinary()` by default
+ * @returns {string | null} an opaque identity, stable while that build is
+ */
+export function ufBinaryIdentity(command = ufBinary()) {
+  const binary = resolveExecutable(command);
+  if (binary == null) return null;
+  try {
+    const stats = statSync(binary);
+    if (!stats.isFile()) return null;
+    accessSync(binary, constants.X_OK);
+    return `${binary}\0${stats.size}\0${stats.mtimeMs}`;
+  } catch {
+    // Named a binary that is not there, or is not one. The caller gets `null`
+    // and stops trusting the cache, which is right: nothing can be compiled
+    // either.
+    return null;
+  }
+}
+
+/**
+ * The file `spawn` will execute for `command`, or `null` when there is none.
+ *
+ * A bare name is searched along PATH the way `execvp` searches for it — the
+ * first regular, executable file wins — so that the identity above describes
+ * the binary that actually runs rather than some other `uf` further down the
+ * list. Getting this wrong is not a slow cache but a silently stale one, which
+ * is why a directory named `uf` is skipped here as `execvp` skips it, rather
+ * than being accepted because `access` says a directory is executable.
+ *
+ * Windows resolves a bare name by rules of its own — `PATHEXT`, the current
+ * directory — which this does not implement. There a bare name is `null` and
+ * the caller falls back to not caching, rather than to caching under the
+ * identity of a file that may not be the one that ran. `UF_BINARY`, which is
+ * how every uf-started host arrives here, is an absolute path on every
+ * platform and never takes this path at all.
+ */
+function resolveExecutable(command) {
+  // A path is taken as given — `spawn` will execute exactly it — and
+  // `ufBinaryIdentity` applies the file-and-executable test to the result
+  // either way, so a path that is a directory or is not executable is no more
+  // trusted than a bare name that resolves to one.
+  if (path.basename(command) !== command) return command;
+  for (const directory of (process.env.PATH ?? "").split(path.delimiter)) {
+    if (directory === "") continue;
+    const candidate = path.join(directory, command);
+    try {
+      if (!statSync(candidate).isFile()) continue;
+      accessSync(candidate, constants.X_OK);
+      return candidate;
+    } catch {
+      // Not in this directory. Keep looking, exactly as the shell would.
+    }
+  }
+  return null;
+}
+
 /**
  * An error the transform reported for one module, with its position when
  * the parser or the lowering rules gave one.
@@ -86,6 +173,7 @@ export class TransformError extends Error {
 export class TransformService {
   #child;
   #pending = [];
+  #identity;
   #failure = null;
 
   /**
@@ -96,6 +184,14 @@ export class TransformService {
   constructor(options = {}) {
     const command = options.command ?? ufBinary();
     const root = options.root ?? process.cwd();
+    // Read before the spawn and kept: this is the identity of the build that
+    // answers every request this service ever serves, because a child goes on
+    // executing the binary it started from however many times that file is
+    // rewritten underneath it. Anything written to disk from an answer of
+    // this service belongs under *this* identity — a caller that stat'd the
+    // binary earlier and wrote under that would file build B's output under
+    // build A's name, which is the original defect with a smaller window.
+    this.#identity = ufBinaryIdentity(command);
     this.#child = spawn(command, ["--cwd", root, "transform"], {
       stdio: ["pipe", "pipe", "inherit"],
     });
@@ -166,6 +262,21 @@ export class TransformService {
     });
   }
 
+  /**
+   * The build of `uf` this service's child is executing, or `null` when that
+   * could not be established.
+   *
+   * Read once, before the spawn, and never again: the child goes on executing
+   * the binary it started from however many times that file is rewritten
+   * underneath it. Anything kept from an answer of this service belongs under
+   * this identity and not under whatever the file says now.
+   *
+   * @returns {string | null}
+   */
+  get identity() {
+    return this.#identity;
+  }
+
   /** Stop the process. Outstanding requests are rejected. */
   close() {
     this.#child.stdin.end();