@akanjs/devkit 2.4.1-rc.2 → 2.4.1-rc.4

package/frontendBuild/hmrWatcher.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import type { Logger } from "akanjs/common";
 import type { ChangeBatch, ChangeKind } from "akanjs/server";
 import { HmrChangeClassifier } from "./hmrChangeClassifier";
+import { SourceMtimeIndex } from "./sourceMtimeIndex";
 
 export type { ChangeBatch, ChangeKind };
 
@@ -11,6 +12,11 @@ export interface WatcherOptions {
   debounceMs?: number;
   logger: Logger;
   onBatch: (batch: ChangeBatch) => void | Promise<void>;
+  /**
+   * Delay before re-checking mtimes for changes `fs.watch` never reported. Must clear Bun's ~200ms
+   * coalescing window, or a write late in the same window as a delivered event stays invisible.
+   */
+  verifyDelayMs?: number;
 }
 
 /**
@@ -18,27 +24,55 @@ export interface WatcherOptions {
  * classification coarse (`code` / `css` / `config`) so the orchestrator can
  * decide whether a full rebuild-and-reload or a narrower action (e.g. CSS
  * hot-swap) is sufficient.
+ *
+ * Bun's event payloads are treated as a hint, not the answer: its recursive `fs.watch` reports about one
+ * path per coalescing window and silently discards the rest, so what changed is resolved against a
+ * `SourceMtimeIndex` instead (see that class for the measurements). Events still drive *when* to look,
+ * which is what keeps this cheap — Bun does reliably deliver at least one event per window.
  */
 export class HmrWatcher {
   readonly #roots: string[];
   readonly #debounceMs: number;
+  readonly #verifyDelayMs: number;
   readonly #onBatch: WatcherOptions["onBatch"];
   readonly #logger: Logger;
   readonly #watchers: fs.FSWatcher[] = [];
   readonly #pending = new Map<string, Exclude<ChangeKind, "ignore">>();
+  /** Paths the watcher's own events named since the last batch — diagnostics only, see `#queue`. */
+  readonly #hinted = new Set<string>();
   readonly #classifier = new HmrChangeClassifier();
+  readonly #index: SourceMtimeIndex;
   #timer: ReturnType<typeof setTimeout> | null = null;
+  #verifyTimer: ReturnType<typeof setTimeout> | null = null;
   #stopped = false;
   #flushing = false;
+  #unreportedChanges = 0;
+  #reportedCompensating = false;
+  #reportedGaps = "";
 
   constructor(opts: WatcherOptions) {
     this.#roots = [...new Set(opts.roots.map((r) => path.resolve(r)))];
     this.#debounceMs = opts.debounceMs ?? 80;
+    this.#verifyDelayMs = opts.verifyDelayMs ?? 250;
     this.#onBatch = opts.onBatch;
     this.#logger = opts.logger;
+    this.#index = new SourceMtimeIndex({ roots: this.#roots, classifier: this.#classifier });
   }
 
-  start(): void {
+  /**
+   * How many changes the mtime scan found that `fs.watch` never reported. Non-zero means this watcher is
+   * compensating for the Bun defect rather than the defect being absent, which is the number to watch if
+   * it is ever fixed upstream.
+   */
+  get unreportedChanges(): number {
+    return this.#unreportedChanges;
+  }
+
+  /**
+   * Watchers are installed before the baseline is taken, so an edit made during priming is reported by
+   * the event side even though the baseline already reflects it.
+   */
+  async start(): Promise<void> {
     for (const root of this.#roots) {
       try {
         const w = fs.watch(root, { recursive: true, persistent: false }, (_event, filename) => {
@@ -52,11 +86,23 @@ export class HmrWatcher {
         this.#logger.error(`[hmr] failed to watch ${root}: ${(err as Error).message}`);
       }
     }
+    try {
+      await this.#index.prime();
+      // Before the first batch, because a root that is unreadable at boot blinds the whole session and
+      // waiting for a save to surface it means waiting for a save that never rebuilds.
+      this.#reportCoverageGaps();
+      this.#logger.verbose(`[hmr] tracking ${this.#index.trackedFileCount} source files for change verification`);
+    } catch (err) {
+      this.#logger.error(
+        `[hmr] mtime index unavailable; falling back to watcher events alone, which drop concurrent saves: ${(err as Error).message}`,
+      );
+    }
   }
 
   stop(): void {
     this.#stopped = true;
     if (this.#timer) clearTimeout(this.#timer);
+    if (this.#verifyTimer) clearTimeout(this.#verifyTimer);
     for (const w of this.#watchers) {
       try {
         w.close();
@@ -66,10 +112,46 @@ export class HmrWatcher {
     }
   }
 
+  /**
+   * Adopt writes the batch handler made itself (regenerated barrels, inserted imports) so the
+   * verification scan does not spend a second generation rebuilding content this one already consumed.
+   */
+  async absorb(paths: string[]): Promise<void> {
+    if (paths.length === 0) return;
+    await this.#index.absorb(paths);
+  }
+
+  /**
+   * An event says only *that* something happened, never reliably *what*. So with a baseline in hand it is
+   * used purely to decide when to look, and the mtime scan names the files.
+   *
+   * Taking the payload as well would double-report: Bun does deliver a real event for some of the paths a
+   * scan has already emitted, and adding it back here produced a second batch for the same save — one more
+   * generation and one more build for no change.
+   */
   #queue(abs: string): void {
     const kind = this.#classifier.classify(abs);
-    if (kind === "ignore") return;
-    this.#pending.set(abs, kind);
+    if (!this.#index.primed) {
+      // Still priming, or priming failed: the payload is the only signal there is.
+      if (kind === "ignore") return;
+      this.#pending.set(abs, kind);
+      this.#scheduleFlush();
+      return;
+    }
+    // An ignored path still means a window happened. That is the shape of the original bug — every build
+    // ends in a burst under `.akan/`, and the burst is what Bun reports instead of the save beside it.
+    // Rescheduling on each one folds a build's whole burst into a single scan once it goes quiet.
+    if (kind === "ignore") {
+      this.#scheduleVerify();
+      return;
+    }
+    // Recorded only so `unreportedChanges` can tell which changes the events did name; it never decides
+    // what is in a batch.
+    this.#hinted.add(abs);
+    this.#scheduleFlush();
+  }
+
+  #scheduleFlush(): void {
     if (this.#flushing) return;
     if (this.#timer) clearTimeout(this.#timer);
     this.#timer = setTimeout(() => this.#flush(), this.#debounceMs);
@@ -77,17 +159,20 @@ export class HmrWatcher {
 
   #flush(): void {
     this.#timer = null;
-    if (this.#stopped || this.#pending.size === 0 || this.#flushing) return;
+    if (this.#stopped || this.#flushing) return;
     void this.#drain();
   }
 
   async #drain(): Promise<void> {
     this.#flushing = true;
     try {
-      while (!this.#stopped && this.#pending.size > 0) {
+      while (!this.#stopped) {
+        await this.#mergeDetectedChanges();
+        if (this.#pending.size === 0) break;
         const files = Array.from(this.#pending.keys());
         const kinds = new Set(this.#pending.values());
         this.#pending.clear();
+        this.#hinted.clear();
         try {
           await this.#onBatch({ files, kinds });
         } catch (e) {
@@ -97,6 +182,92 @@ export class HmrWatcher {
     } finally {
       this.#flushing = false;
       if (!this.#stopped && this.#pending.size > 0) this.#timer = setTimeout(() => this.#flush(), this.#debounceMs);
+      else this.#scheduleVerify();
     }
   }
+
+  /**
+   * Fold in everything the mtime index has seen change, since a delivered event names at most one of the
+   * paths that moved in its window.
+   */
+  async #mergeDetectedChanges(): Promise<void> {
+    const detected = await this.#index.collectChanges().catch((err) => {
+      this.#logger.error(`[hmr] mtime scan failed: ${(err as Error).message}`);
+      return [] as string[];
+    });
+    this.#reportCoverageGaps();
+    let unreported = 0;
+    for (const abs of detected) {
+      const kind = this.#classifier.classify(abs);
+      if (kind === "ignore") continue;
+      if (!this.#hinted.has(abs)) unreported += 1;
+      this.#pending.set(abs, kind);
+    }
+    if (unreported === 0) return;
+    this.#unreportedChanges += unreported;
+    // Once at info, so it is visible that the watcher is compensating rather than the defect being absent;
+    // per-batch detail stays at verbose because a save-all trips this on every save.
+    if (!this.#reportedCompensating) {
+      this.#reportedCompensating = true;
+      this.#logger.info(
+        `[hmr] recovered ${unreported} change(s) that fs.watch did not report; Bun coalesces concurrent saves and drops all but one, so changes are resolved by mtime`,
+      );
+    }
+    this.#logger.verbose(
+      `[hmr] mtime scan found ${unreported} change(s) fs.watch never reported (${this.#unreportedChanges} total)`,
+    );
+  }
+
+  /**
+   * Say so when part of the tree cannot be read, and say so again when it recovers.
+   *
+   * A blind spot here means edits under that path are not rebuilt at all, which is indistinguishable from
+   * the dev server being broken. Keyed on the gap list itself so a persistent failure logs once rather than
+   * once per save, while a *different* gap appearing still gets its own line.
+   */
+  #reportCoverageGaps(): void {
+    const gaps = this.#index.coverageGaps;
+    const key = gaps
+      .map((gap) => `${gap.code}:${gap.path}`)
+      .sort()
+      .join("|");
+    if (key === this.#reportedGaps) return;
+    this.#reportedGaps = key;
+    if (gaps.length === 0) {
+      this.#logger.info("[hmr] all watch roots readable again; change detection is complete");
+      return;
+    }
+    const shown = gaps
+      .slice(0, 3)
+      .map((gap) => `${gap.path} (${gap.code})`)
+      .join(", ");
+    const rest = gaps.length > 3 ? ` and ${gaps.length - 3} more` : "";
+    this.#logger.warn(
+      `[hmr] cannot read ${gaps.length} path(s), so edits underneath them will not rebuild: ${shown}${rest}`,
+    );
+  }
+
+  /**
+   * One scan after the coalescing window closes. A write that lands in the same window as an already
+   * delivered event produces no further event of its own, so nothing else would ever look for it.
+   */
+  #scheduleVerify(): void {
+    if (this.#stopped || this.#verifyDelayMs <= 0) return;
+    if (this.#verifyTimer) clearTimeout(this.#verifyTimer);
+    this.#verifyTimer = setTimeout(() => {
+      this.#verifyTimer = null;
+      void this.#verify();
+    }, this.#verifyDelayMs);
+  }
+
+  /**
+   * Terminates rather than looping: a scan that finds nothing schedules nothing, and the build writes its
+   * artifacts under `.akan/` — which the classifier ignores — while codegen writes are content-guarded,
+   * so a rebuild does not move a tracked mtime.
+   */
+  async #verify(): Promise<void> {
+    if (this.#stopped || this.#flushing) return;
+    await this.#mergeDetectedChanges();
+    if (this.#pending.size > 0) await this.#drain();
+  }
 }

package/frontendBuild/index.ts

@@ -16,6 +16,7 @@ export * from "./pagesEntrySourceGenerator";
 export * from "./precompressArtifacts";
 export * from "./routeClientBuilder";
 export * from "./routesManifestArtifactSerializer";
+export * from "./sourceMtimeIndex";
 export * from "./ssrBaseArtifactBuilder";
 export * from "./vendorSpecifiers";
 export * from "./watchRootResolver";

package/frontendBuild/sourceMtimeIndex.test.ts

@@ -0,0 +1,280 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { chmod, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { SourceMtimeIndex } from "./sourceMtimeIndex";
+
+const roots: string[] = [];
+
+const makeRoot = async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "akan-mtime-index-"));
+  roots.push(root);
+  return root;
+};
+
+const seed = async (root: string, rel: string, content = "export const x = 1;\n") => {
+  const abs = path.join(root, rel);
+  await mkdir(path.dirname(abs), { recursive: true });
+  await writeFile(abs, content);
+  return abs;
+};
+
+/**
+ * mtime comparison needs the write to land on a different timestamp than the baseline. APFS records
+ * nanoseconds so same-millisecond writes normally still differ, but size is compared too and a
+ * same-length rewrite inside one tick would tie both — so tests vary content length rather than sleep.
+ */
+const rewrite = (abs: string, marker: string) => writeFile(abs, `export const x = ${marker};\n`);
+
+/**
+ * `rm` cannot traverse a directory a test left unreadable, and a throw here fails the *next* test rather
+ * than the one that caused it — so permissions are restored on the way down before giving up.
+ */
+const forceRemove = async (target: string): Promise<void> => {
+  await rm(target, { recursive: true, force: true }).catch(async () => {
+    await chmod(target, 0o755).catch(() => undefined);
+    const entries = await readdir(target, { withFileTypes: true }).catch(() => []);
+    for (const entry of entries) if (entry.isDirectory()) await forceRemove(path.join(target, entry.name));
+    await rm(target, { recursive: true, force: true });
+  });
+};
+
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => forceRemove(root)));
+});
+
+describe("SourceMtimeIndex", () => {
+  test("reports nothing on a quiet tree", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    expect(index.trackedFileCount).toBe(1);
+    expect(await index.collectChanges()).toEqual([]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("reports every file of a save-all, which is what fs.watch loses", async () => {
+    const root = await makeRoot();
+    const files = await Promise.all([0, 1, 2, 3, 4].map((i) => seed(root, `lib/File${i}.ts`)));
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    // No gaps: exactly the burst Bun collapses to a single reported path.
+    for (const [i, abs] of files.entries()) await rewrite(abs, `${i}00`);
+
+    expect((await index.collectChanges()).sort()).toEqual([...files].sort());
+  });
+
+  test("reports a change once, then stops reporting it", async () => {
+    const root = await makeRoot();
+    const abs = await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    await rewrite(abs, "222");
+    expect(await index.collectChanges()).toEqual([abs]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("reports a created file, found through its directory's mtime", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    const created = await seed(root, "lib/b.ts");
+    expect(await index.collectChanges()).toEqual([created]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("reports a created directory's files", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    const created = await seed(root, "lib/user/user.constant.ts");
+    expect(await index.collectChanges()).toEqual([created]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("reports a deleted file and a deleted directory's files", async () => {
+    const root = await makeRoot();
+    const kept = await seed(root, "lib/a.ts");
+    const removed = await seed(root, "lib/gone/b.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    await rm(path.join(root, "lib/gone"), { recursive: true, force: true });
+    expect(await index.collectChanges()).toEqual([removed]);
+    expect(await index.collectChanges()).toEqual([]);
+    expect(await index.collectChanges()).not.toContain(kept);
+  });
+
+  test("ignores build output and node_modules", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+    expect(index.trackedFileCount).toBe(1);
+
+    await seed(root, ".akan/artifact/server/pages-1.js");
+    await seed(root, "node_modules/dep/index.ts");
+    expect(await index.collectChanges()).toEqual([]);
+    expect(index.trackedFileCount).toBe(1);
+  });
+
+  test("ignores files no classifier kind applies to", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    await seed(root, "public/logo.svg", "<svg/>");
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("absorb adopts a write instead of reporting it", async () => {
+    const root = await makeRoot();
+    const abs = await seed(root, "lib/index.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    await rewrite(abs, "333");
+    await index.absorb([abs]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("absorb of an unknown path does not start tracking a change", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    const created = await seed(root, "lib/generated.ts");
+    await index.absorb([created]);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  test("counts a file once when a root is nested inside another root", async () => {
+    const root = await makeRoot();
+    await seed(root, "page/_index.tsx");
+    const nested = new SourceMtimeIndex({ roots: [root, path.join(root, "page")] });
+    await nested.prime();
+    const flat = new SourceMtimeIndex({ roots: [root] });
+    await flat.prime();
+
+    expect(nested.trackedFileCount).toBe(flat.trackedFileCount);
+  });
+
+  test("concurrent scans do not invent a change", async () => {
+    const root = await makeRoot();
+    await Promise.all([0, 1, 2, 3, 4].map((i) => seed(root, `lib/File${i}.ts`)));
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    // Overlapping scans used to corrupt each other's view of the baseline: one pruned an entry the other
+    // had already snapshotted, and the path came back as a change nothing had written.
+    const rounds = await Promise.all([1, 2, 3, 4].map(() => index.collectChanges()));
+    expect(rounds.flat()).toEqual([]);
+  });
+
+  test("concurrent scans report a real change exactly once between them", async () => {
+    const root = await makeRoot();
+    const abs = await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+    await index.prime();
+
+    await rewrite(abs, "444");
+    const rounds = await Promise.all([1, 2, 3].map(() => index.collectChanges()));
+    expect(rounds.flat()).toEqual([abs]);
+  });
+
+  test("reports nothing before priming rather than treating the tree as new", async () => {
+    const root = await makeRoot();
+    await seed(root, "lib/a.ts");
+    const index = new SourceMtimeIndex({ roots: [root] });
+
+    expect(index.primed).toBe(false);
+    expect(await index.collectChanges()).toEqual([]);
+  });
+
+  describe("a directory it cannot read", () => {
+    /**
+     * `chmod 000` does not stop root, so these would assert the opposite of what they mean when the suite
+     * runs as root (CI containers commonly do).
+     */
+    const asRoot = process.getuid?.() === 0;
+
+    test.skipIf(asRoot)("is reported as a gap instead of silently skipped", async () => {
+      const root = await makeRoot();
+      await seed(root, "open/a.ts");
+      const hidden = await seed(root, "locked/b.ts");
+      await chmod(path.dirname(hidden), 0o000);
+
+      const index = new SourceMtimeIndex({ roots: [root] });
+      await index.prime();
+
+      // Priming still succeeds — it just cannot see everything, and that is the part that must be said out
+      // loud rather than inferred from a file count nobody is checking.
+      expect(index.primed).toBe(true);
+      expect(index.trackedFileCount).toBe(1);
+      expect(index.coverageGaps).toEqual([{ path: path.dirname(hidden), code: "EACCES" }]);
+    });
+
+    test.skipIf(asRoot)("becomes visible again once it is readable, and clears the gap", async () => {
+      const root = await makeRoot();
+      const hidden = await seed(root, "locked/b.ts");
+      const locked = path.dirname(hidden);
+      await chmod(locked, 0o000);
+      const index = new SourceMtimeIndex({ roots: [root] });
+      await index.prime();
+
+      await chmod(locked, 0o755);
+
+      // Before the fix this stayed empty forever: the directory had been forgotten, so nothing re-stat'd
+      // it, and its parent's mtime never moves when a child merely becomes readable again.
+      expect(await index.collectChanges()).toEqual([hidden]);
+      expect(index.coverageGaps).toEqual([]);
+      await rewrite(hidden, "999");
+      expect(await index.collectChanges()).toEqual([hidden]);
+    });
+
+    test.skipIf(asRoot)("does not report the files under it as deleted while it is unreadable", async () => {
+      const root = await makeRoot();
+      const abs = await seed(root, "locked/b.ts");
+      const locked = path.dirname(abs);
+      const index = new SourceMtimeIndex({ roots: [root] });
+      await index.prime();
+      expect(index.trackedFileCount).toBe(1);
+
+      await chmod(locked, 0o000);
+
+      // A phantom deletion would rebuild for nothing and, worse, drop the file so a later real edit to it
+      // goes unreported.
+      expect(await index.collectChanges()).toEqual([]);
+      expect(index.trackedFileCount).toBe(1);
+      expect(index.coverageGaps.map((gap) => gap.code)).toEqual(["EACCES"]);
+
+      await chmod(locked, 0o755);
+      await rewrite(abs, "1234");
+      expect(await index.collectChanges()).toEqual([abs]);
+      expect(index.coverageGaps).toEqual([]);
+    });
+
+    test("is still forgotten when it is genuinely deleted, with no gap left behind", async () => {
+      const root = await makeRoot();
+      const abs = await seed(root, "lib/a.ts");
+      const index = new SourceMtimeIndex({ roots: [root] });
+      await index.prime();
+
+      await rm(path.dirname(abs), { recursive: true, force: true });
+
+      expect(await index.collectChanges()).toEqual([abs]);
+      expect(index.coverageGaps).toEqual([]);
+      expect(await index.collectChanges()).toEqual([]);
+    });
+  });
+});