@decohere/flux 0.0.1-dev.110 → 0.0.1-dev.111

package/README.md

@@ -214,7 +214,7 @@ Only source files (`.ts`) that a served entry imports trigger a reboot.
 
 **Memory grows over a long session.** Each reboot imports changed modules under a new `?version=` URL, and Node's ES module cache never releases the old versions. That's fine for a working day; if a session runs for days with thousands of reboots, restart it.
 
-Flux uses **Forge’s compiler**, with one SWC config per project built once per run. SWC leaves imports as written; Flux's resolve hook maps each one to its `.ts` source: `@scope/pkg` and `@scope/pkg/sub` through the Mesh aliases (and tsconfig `paths`), `./file.js` and `./file` to `file.ts`, and `./dir` to `dir/index.ts`. Doing it in one place, instead of in SWC (`jsc.paths`, `resolveFully`) and again in the hook, made loading about 20% faster once Node and SWC are warm (`pnpm bench flux-load`, #513). That is a debug rule so you can edit a library and hit HMR without a pre-build or `node_modules` workspace symlink. Release still resolves through real `node_modules` (published `exports` / `main`).
+Flux uses **Forge’s compiler**, with one SWC config per project built once per run. SWC leaves imports as written; Flux's resolve hook maps each one to its `.ts` source: `@scope/pkg` and `@scope/pkg/sub` through the Mesh aliases (and tsconfig `paths`), `./file.js` and `./file` to `file.ts`, and `./dir` to `dir/index.ts`. Doing it in one place, instead of in SWC (`jsc.paths`, `resolveFully`) and again in the hook, made loading about 20% faster once Node and SWC are warm (`pnpm bench flux-load`, #513). Before Node imports an entry, Flux compiles it and the files it imports, following their imports out from the entry, in parallel on SWC's thread pool (#661). The load hook is synchronous and would otherwise compile one file at a time; files the app never imports, such as tests, are not compiled. A file that fails to compile there is compiled again when Node loads it, and that load fails with the file's name, as before. That is a debug rule so you can edit a library and hit HMR without a pre-build or `node_modules` workspace symlink. Release still resolves through real `node_modules` (published `exports` / `main`).
 
 Why aliases, and a tree of where a symlink misses `src/index.ts`: [Forge — debug path aliases](../forge/README.md#debug-path-aliases-not-node_modules-links).
 

package/commonjs/src/serve.command.js

@@ -25,6 +25,9 @@ class ServeCommand extends _cli.DecohereCommand {
         this.addOption(new _cli.ProjectOption());
         this.option("--reboot-debounce-ms <ms>", "Quiet window in milliseconds after the last affecting save before a reboot", (value)=>(0, _hotreloadserver.resolveRebootDebounceMs)(Number(value)), _hotreloadserver.DEFAULT_REBOOT_DEBOUNCE_MS);
     }
+    runsUntilStopped() {
+        return true;
+    }
     /**
 	 * Commander passes positional args before options when `[slices...]` is declared.
 	 * The CLI host ModuleFactory expects `(opts, cmd)` — normalize here.

package/commonjs/src/source-module-loader.js

@@ -17,6 +17,7 @@ const _mesh = require("@decohere/mesh");
 const _io = require("@decohere/nexus/io");
 const _scribe = require("@decohere/scribe");
 const _rewriteservespecifier = require("./rewrite-serve-specifier.js");
+const _staticimports = require("./static-imports.js");
 function _interop_require_default(obj) {
     return obj && obj.__esModule ? obj : {
         default: obj
@@ -78,6 +79,9 @@ class SourceModuleLoader {
     buildConfig = "debug";
     fileInfoMap = new Map();
     transpileCache = new Map();
+    /** Bumped when a file's cached code is dropped, so an up-front compile already running can't put it back. */ transpileVersions = new Map();
+    /** The source files each entry's walk reached, so a reboot only walks out from what changed. */ reachedFrom = new Map();
+    /** Up-front compiles in flight, shared by entries booting at the same time. */ compilingAhead = new Map();
     /** SWC options per project, built on first use and kept until the next `configure()`. */ swcConfigs = new Map();
     aliasPaths = {};
     aliasBaseUrl = _nodepath.default.resolve(".");
@@ -102,7 +106,9 @@ class SourceModuleLoader {
     }
     async configure(options) {
         this.disposed = false;
-        this.transpileCache.clear();
+        // A new Map, not clear(): an up-front compile from before can't write into this one.
+        this.transpileCache = new Map();
+        this.reachedFrom = new Map();
         this.swcConfigs.clear();
         this.buildConfig = options.buildConfig;
         this.workspace = options.workspace ?? await (0, _forge.createWorkspace)();
@@ -149,12 +155,14 @@ class SourceModuleLoader {
         });
     }
     async importEntry(absolutePath) {
+        await this.compileAhead(_nodepath.default.resolve(absolutePath));
         const url = this.versionFileUrl(_io.io.pathToFileUrl(_nodepath.default.resolve(absolutePath)));
         return Promise.resolve(url).then((p)=>/*#__PURE__*/ _interop_require_wildcard(require(p)));
     }
     invalidateTranspile(...absolutePaths) {
         for (const fullPath of absolutePaths){
             this.transpileCache.delete(fullPath);
+            this.transpileVersions.set(fullPath, (this.transpileVersions.get(fullPath) ?? 0) + 1);
             this.transpiler.invalidateContentCache(new _io.FileEx({
                 fullPath
             }));
@@ -162,7 +170,8 @@ class SourceModuleLoader {
     }
     dispose() {
         this.disposed = true;
-        this.transpileCache.clear();
+        this.transpileCache = new Map();
+        this.reachedFrom = new Map();
         this.swcConfigs.clear();
         this.fileInfoMap.clear();
         this.workspace?.[Symbol.dispose]();
@@ -181,6 +190,71 @@ class SourceModuleLoader {
         }));
         return new Map(entries.flat());
     }
+    /**
+	 * Compiles the entry and the source files it imports, before Node imports it (#661). The load
+	 * hook is synchronous, so it would compile one file at a time on the main thread; async SWC
+	 * runs on the thread pool. Walks out from the entry, compiling each file's imports in parallel,
+	 * so files the app never imports (tests, other apps) are not compiled. A reboot walks out only
+	 * from the files it reached before that are no longer cached (the saved ones), so their new
+	 * imports are found without visiting the rest of the graph again.
+	 *
+	 * A file that fails here is left out of the cache instead of failing the boot: the load hook
+	 * then compiles it again and throws as before, naming the file. The walk is a hint; whatever
+	 * it misses is compiled by the load hook when Node gets to it.
+	 */ async compileAhead(entryPath) {
+        const reached = this.reachedFrom.get(entryPath);
+        const starts = reached == null ? [
+            entryPath
+        ] : [
+            ...reached
+        ].filter((fullPath)=>!this.transpileCache.has(fullPath));
+        const seen = new Set(reached);
+        for (const start of starts)seen.delete(start);
+        const visit = async (fullPath)=>{
+            if (seen.has(fullPath)) return;
+            seen.add(fullPath);
+            const code = await this.compiledAhead(fullPath);
+            if (code == null) return;
+            const parentUrl = (0, _nodeurl.pathToFileURL)(fullPath).toString();
+            await Promise.all(_staticimports.staticImports.scan(code).map((specifier)=>{
+                const target = this.sourcePathOf(specifier, parentUrl);
+                return target == null ? undefined : visit(target);
+            }));
+        };
+        await Promise.all(starts.map(visit));
+        this.reachedFrom.set(entryPath, seen);
+    }
+    /** The file's compiled code, from the cache or compiled now; undefined if it isn't a source. */ compiledAhead(fullPath) {
+        const fileInfo = this.fileInfoMap.get(fullPath);
+        if (fileInfo == null) return Promise.resolve(undefined);
+        const cached = this.transpileCache.get(fullPath);
+        if (cached != null) return Promise.resolve(cached);
+        const inFlight = this.compilingAhead.get(fullPath);
+        if (inFlight != null) return inFlight;
+        const version = this.transpileVersions.get(fullPath);
+        const cache = this.transpileCache;
+        const compiling = this.transpiler.transformFileAsync(new _io.FileEx({
+            fullPath
+        }), _forge.BuildTargets.es6, {}, this.swcConfigFor(fileInfo.project)).then((response)=>{
+            if (response.outcome !== "success") return undefined;
+            // Saved, reconfigured or disposed while compiling: don't cache stale code.
+            const current = !this.disposed && cache === this.transpileCache && version === this.transpileVersions.get(fullPath);
+            if (current) cache.set(fullPath, response.code);
+            return response.code;
+        }).finally(()=>this.compilingAhead.delete(fullPath));
+        this.compilingAhead.set(fullPath, compiling);
+        return compiling;
+    }
+    /** The source file a specifier names, mapped the way the resolve hook maps it. */ sourcePathOf(specifier, parentUrl) {
+        const url = (0, _rewriteservespecifier.rewriteServeSpecifier)({
+            specifier,
+            parentUrl,
+            isWatchedTs: (filePath)=>this.fileInfoMap.has(filePath),
+            resolveBare: (bare)=>_mesh.packageAliases.resolvePath(bare, this.aliasPaths, this.aliasBaseUrl),
+            versionFileUrl: (fileUrl)=>fileUrl
+        });
+        return url == null ? null : (0, _nodeurl.fileURLToPath)(url);
+    }
     loadTs(url) {
         if (!url.startsWith("file:")) return null;
         // fileURLToPath, not URL.pathname: decodes %20 and gives Windows paths, so files

package/commonjs/src/static-imports.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "staticImports", {
+    enumerable: true,
+    get: function() {
+        return staticImports;
+    }
+});
+/**
+ * The module specifiers a compiled ES module names in `import … from`, `export … from`, bare
+ * `import "x"` and literal `import("x")`.
+ *
+ * Only a hint for compiling ahead (#661): a specifier it misses is compiled when Node loads it,
+ * and one it finds by mistake (an import written inside a string) costs one extra compile. So a
+ * scan of SWC's output is enough; it never decides what runs.
+ */ const STATIC = /\b(?:import|export)\s*(?:[\w$*{},\s]+?\s*from\s*)?["']([^"'\n]+)["']/g;
+const DYNAMIC = /\bimport\s*\(\s*["']([^"'\n]+)["']\s*\)/g;
+function scan(code) {
+    const found = new Set();
+    for (const pattern of [
+        STATIC,
+        DYNAMIC
+    ]){
+        for (const match of code.matchAll(pattern)){
+            const specifier = match[1];
+            if (specifier != null) found.add(specifier);
+        }
+    }
+    return [
+        ...found
+    ];
+}
+const staticImports = {
+    scan
+};

package/es6/src/serve.command.js

@@ -15,6 +15,9 @@ export class ServeCommand extends DecohereCommand {
         this.addOption(new ProjectOption());
         this.option("--reboot-debounce-ms <ms>", "Quiet window in milliseconds after the last affecting save before a reboot", (value)=>resolveRebootDebounceMs(Number(value)), DEFAULT_REBOOT_DEBOUNCE_MS);
     }
+    runsUntilStopped() {
+        return true;
+    }
     /**
 	 * Commander passes positional args before options when `[slices...]` is declared.
 	 * The CLI host ModuleFactory expects `(opts, cmd)` — normalize here.

package/es6/src/source-module-loader.js

@@ -6,13 +6,14 @@ function _ts_decorate(decorators, target, key, desc) {
 }
 import { registerHooks } from "node:module";
 import path from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 import { injectable } from "@decohere/crucible/decorators";
 import { BuildTargets, createWorkspace, SwcTranspiler } from "@decohere/forge";
 import { packageAliases } from "@decohere/mesh";
 import { FileEx, io } from "@decohere/nexus/io";
 import { scribe } from "@decohere/scribe";
 import { rewriteServeSpecifier } from "./rewrite-serve-specifier.js";
+import { staticImports } from "./static-imports.js";
 export class SourceModuleLoader {
     logger = scribe.logger(SourceModuleLoader.name);
     transpiler = new SwcTranspiler();
@@ -22,6 +23,9 @@ export class SourceModuleLoader {
     buildConfig = "debug";
     fileInfoMap = new Map();
     transpileCache = new Map();
+    /** Bumped when a file's cached code is dropped, so an up-front compile already running can't put it back. */ transpileVersions = new Map();
+    /** The source files each entry's walk reached, so a reboot only walks out from what changed. */ reachedFrom = new Map();
+    /** Up-front compiles in flight, shared by entries booting at the same time. */ compilingAhead = new Map();
     /** SWC options per project, built on first use and kept until the next `configure()`. */ swcConfigs = new Map();
     aliasPaths = {};
     aliasBaseUrl = path.resolve(".");
@@ -46,7 +50,9 @@ export class SourceModuleLoader {
     }
     async configure(options) {
         this.disposed = false;
-        this.transpileCache.clear();
+        // A new Map, not clear(): an up-front compile from before can't write into this one.
+        this.transpileCache = new Map();
+        this.reachedFrom = new Map();
         this.swcConfigs.clear();
         this.buildConfig = options.buildConfig;
         this.workspace = options.workspace ?? await createWorkspace();
@@ -93,12 +99,14 @@ export class SourceModuleLoader {
         });
     }
     async importEntry(absolutePath) {
+        await this.compileAhead(path.resolve(absolutePath));
         const url = this.versionFileUrl(io.pathToFileUrl(path.resolve(absolutePath)));
         return import(url);
     }
     invalidateTranspile(...absolutePaths) {
         for (const fullPath of absolutePaths){
             this.transpileCache.delete(fullPath);
+            this.transpileVersions.set(fullPath, (this.transpileVersions.get(fullPath) ?? 0) + 1);
             this.transpiler.invalidateContentCache(new FileEx({
                 fullPath
             }));
@@ -106,7 +114,8 @@ export class SourceModuleLoader {
     }
     dispose() {
         this.disposed = true;
-        this.transpileCache.clear();
+        this.transpileCache = new Map();
+        this.reachedFrom = new Map();
         this.swcConfigs.clear();
         this.fileInfoMap.clear();
         this.workspace?.[Symbol.dispose]();
@@ -125,6 +134,71 @@ export class SourceModuleLoader {
         }));
         return new Map(entries.flat());
     }
+    /**
+	 * Compiles the entry and the source files it imports, before Node imports it (#661). The load
+	 * hook is synchronous, so it would compile one file at a time on the main thread; async SWC
+	 * runs on the thread pool. Walks out from the entry, compiling each file's imports in parallel,
+	 * so files the app never imports (tests, other apps) are not compiled. A reboot walks out only
+	 * from the files it reached before that are no longer cached (the saved ones), so their new
+	 * imports are found without visiting the rest of the graph again.
+	 *
+	 * A file that fails here is left out of the cache instead of failing the boot: the load hook
+	 * then compiles it again and throws as before, naming the file. The walk is a hint; whatever
+	 * it misses is compiled by the load hook when Node gets to it.
+	 */ async compileAhead(entryPath) {
+        const reached = this.reachedFrom.get(entryPath);
+        const starts = reached == null ? [
+            entryPath
+        ] : [
+            ...reached
+        ].filter((fullPath)=>!this.transpileCache.has(fullPath));
+        const seen = new Set(reached);
+        for (const start of starts)seen.delete(start);
+        const visit = async (fullPath)=>{
+            if (seen.has(fullPath)) return;
+            seen.add(fullPath);
+            const code = await this.compiledAhead(fullPath);
+            if (code == null) return;
+            const parentUrl = pathToFileURL(fullPath).toString();
+            await Promise.all(staticImports.scan(code).map((specifier)=>{
+                const target = this.sourcePathOf(specifier, parentUrl);
+                return target == null ? undefined : visit(target);
+            }));
+        };
+        await Promise.all(starts.map(visit));
+        this.reachedFrom.set(entryPath, seen);
+    }
+    /** The file's compiled code, from the cache or compiled now; undefined if it isn't a source. */ compiledAhead(fullPath) {
+        const fileInfo = this.fileInfoMap.get(fullPath);
+        if (fileInfo == null) return Promise.resolve(undefined);
+        const cached = this.transpileCache.get(fullPath);
+        if (cached != null) return Promise.resolve(cached);
+        const inFlight = this.compilingAhead.get(fullPath);
+        if (inFlight != null) return inFlight;
+        const version = this.transpileVersions.get(fullPath);
+        const cache = this.transpileCache;
+        const compiling = this.transpiler.transformFileAsync(new FileEx({
+            fullPath
+        }), BuildTargets.es6, {}, this.swcConfigFor(fileInfo.project)).then((response)=>{
+            if (response.outcome !== "success") return undefined;
+            // Saved, reconfigured or disposed while compiling: don't cache stale code.
+            const current = !this.disposed && cache === this.transpileCache && version === this.transpileVersions.get(fullPath);
+            if (current) cache.set(fullPath, response.code);
+            return response.code;
+        }).finally(()=>this.compilingAhead.delete(fullPath));
+        this.compilingAhead.set(fullPath, compiling);
+        return compiling;
+    }
+    /** The source file a specifier names, mapped the way the resolve hook maps it. */ sourcePathOf(specifier, parentUrl) {
+        const url = rewriteServeSpecifier({
+            specifier,
+            parentUrl,
+            isWatchedTs: (filePath)=>this.fileInfoMap.has(filePath),
+            resolveBare: (bare)=>packageAliases.resolvePath(bare, this.aliasPaths, this.aliasBaseUrl),
+            versionFileUrl: (fileUrl)=>fileUrl
+        });
+        return url == null ? null : fileURLToPath(url);
+    }
     loadTs(url) {
         if (!url.startsWith("file:")) return null;
         // fileURLToPath, not URL.pathname: decodes %20 and gives Windows paths, so files

package/es6/src/static-imports.js

@@ -0,0 +1,27 @@
+/**
+ * The module specifiers a compiled ES module names in `import … from`, `export … from`, bare
+ * `import "x"` and literal `import("x")`.
+ *
+ * Only a hint for compiling ahead (#661): a specifier it misses is compiled when Node loads it,
+ * and one it finds by mistake (an import written inside a string) costs one extra compile. So a
+ * scan of SWC's output is enough; it never decides what runs.
+ */ const STATIC = /\b(?:import|export)\s*(?:[\w$*{},\s]+?\s*from\s*)?["']([^"'\n]+)["']/g;
+const DYNAMIC = /\bimport\s*\(\s*["']([^"'\n]+)["']\s*\)/g;
+function scan(code) {
+    const found = new Set();
+    for (const pattern of [
+        STATIC,
+        DYNAMIC
+    ]){
+        for (const match of code.matchAll(pattern)){
+            const specifier = match[1];
+            if (specifier != null) found.add(specifier);
+        }
+    }
+    return [
+        ...found
+    ];
+}
+export const staticImports = {
+    scan
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decohere/flux",
-  "version": "0.0.1-dev.110",
+  "version": "0.0.1-dev.111",
   "description": "Hot module reload server for Node.js ESM applications",
   "keywords": [
     "typescript",
@@ -25,11 +25,11 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
-    "@decohere/crucible": "0.0.1-dev.110",
-    "@decohere/forge": "0.0.1-dev.110",
-    "@decohere/mesh": "0.0.1-dev.110",
-    "@decohere/nexus": "0.0.1-dev.110",
-    "@decohere/scribe": "0.0.1-dev.110",
+    "@decohere/crucible": "0.0.1-dev.111",
+    "@decohere/forge": "0.0.1-dev.111",
+    "@decohere/mesh": "0.0.1-dev.111",
+    "@decohere/nexus": "0.0.1-dev.111",
+    "@decohere/scribe": "0.0.1-dev.111",
     "commander": "^13.1.0",
     "rxjs": "^7.8.2"
   },

package/types/src/serve.command.d.ts

@@ -3,6 +3,7 @@ import { ServeAction } from "./serve.action.js";
 export declare class ServeCommand extends DecohereCommand {
     get actions(): (typeof ServeAction)[];
     constructor(name: string);
+    runsUntilStopped(): boolean;
     /**
      * Commander passes positional args before options when `[slices...]` is declared.
      * The CLI host ModuleFactory expects `(opts, cmd)` — normalize here.

package/types/src/source-module-loader.d.ts

@@ -30,6 +30,12 @@ export declare class SourceModuleLoader {
     private buildConfig;
     private fileInfoMap;
     private transpileCache;
+    /** Bumped when a file's cached code is dropped, so an up-front compile already running can't put it back. */
+    private transpileVersions;
+    /** The source files each entry's walk reached, so a reboot only walks out from what changed. */
+    private reachedFrom;
+    /** Up-front compiles in flight, shared by entries booting at the same time. */
+    private compilingAhead;
     /** SWC options per project, built on first use and kept until the next `configure()`. */
     private swcConfigs;
     private aliasPaths;
@@ -47,6 +53,23 @@ export declare class SourceModuleLoader {
     invalidateTranspile(...absolutePaths: string[]): void;
     dispose(): void;
     private buildFileInfoMap;
+    /**
+     * Compiles the entry and the source files it imports, before Node imports it (#661). The load
+     * hook is synchronous, so it would compile one file at a time on the main thread; async SWC
+     * runs on the thread pool. Walks out from the entry, compiling each file's imports in parallel,
+     * so files the app never imports (tests, other apps) are not compiled. A reboot walks out only
+     * from the files it reached before that are no longer cached (the saved ones), so their new
+     * imports are found without visiting the rest of the graph again.
+     *
+     * A file that fails here is left out of the cache instead of failing the boot: the load hook
+     * then compiles it again and throws as before, naming the file. The walk is a hint; whatever
+     * it misses is compiled by the load hook when Node gets to it.
+     */
+    private compileAhead;
+    /** The file's compiled code, from the cache or compiled now; undefined if it isn't a source. */
+    private compiledAhead;
+    /** The source file a specifier names, mapped the way the resolve hook maps it. */
+    private sourcePathOf;
     private loadTs;
     /**
      * The project's SWC options, built once: rebuilding them for every file was about a quarter

package/types/src/static-imports.d.ts

@@ -0,0 +1,5 @@
+declare function scan(code: string): string[];
+export declare const staticImports: {
+    scan: typeof scan;
+};
+export {};