@matter/tools 0.11.0-alpha.0-20241005-e3e4e4a7a

package/src/building/cli.ts

@@ -0,0 +1,89 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { Builder, Target } from "./builder.js";
+import { Graph } from "./graph.js";
+import { Project } from "./project.js";
+import { syncAllTsconfigs } from "./tsconfig.js";
+
+enum Mode {
+    BuildProject,
+    BuildProjectWithDependencies,
+    BuildWorkspace,
+    DisplayGraph,
+    SyncTsconfigs,
+}
+
+export async function main(argv = process.argv) {
+    const targets = Array<Target>();
+    let mode = Mode.BuildProject;
+
+    const args = await yargs(hideBin(argv))
+        .usage("Builds packages adhering to matter.js standards.")
+        .option("prefix", { alias: "p", default: ".", type: "string", describe: "specify build directory" })
+        .option("clean", { alias: "c", default: false, type: "boolean", describe: "clean before build" })
+        .option("workspaces", { alias: "w", default: false, type: "boolean", describe: "build all workspace packages" })
+        .option("dependencies", { alias: "d", default: false, type: "boolean", describe: "build dependencies" })
+        .command("*", "build types and both JS files", () => {})
+        .command("clean", "remove build and dist directories", () => targets.push(Target.clean))
+        .command("types", "build type definitions", () => targets.push(Target.types))
+        .command("esm", "build JS (ES6 modules)", () => targets.push(Target.esm))
+        .command("cjs", "build JS (CommonJS modules)", () => targets.push(Target.cjs))
+        .command("graph", "display the workspace graph", () => (mode = Mode.DisplayGraph))
+        .command("tsconfigs", "sync all tsconfigs with package.json", () => (mode = Mode.SyncTsconfigs))
+        .wrap(Math.min(process.stdout.columns, 80))
+        .strict().argv;
+
+    if (mode === Mode.BuildProject) {
+        if (args.workspaces) {
+            mode = Mode.BuildWorkspace;
+        } else if (args.dependencies) {
+            mode = Mode.BuildProjectWithDependencies;
+        }
+    }
+
+    function builder(graph?: Graph) {
+        return new Builder({ ...args, targets: [...targets], graph });
+    }
+
+    switch (mode as Mode) {
+        case Mode.BuildProject:
+            const project = new Project(args.prefix);
+            await builder().build(project);
+            break;
+
+        case Mode.BuildProjectWithDependencies:
+            {
+                const graph = await Graph.forProject(args.prefix);
+                if (graph === undefined) {
+                    throw new Error(`Cannot build with dependencies because ${args.prefix} is not in a workspace`);
+                }
+                await graph.build(builder(graph));
+            }
+            break;
+
+        case Mode.BuildWorkspace:
+            {
+                const graph = await Graph.load();
+                await syncAllTsconfigs(graph);
+                await graph.build(builder());
+            }
+            break;
+
+        case Mode.DisplayGraph:
+            (await Graph.load()).display();
+            break;
+
+        case Mode.SyncTsconfigs:
+            {
+                const graph = await Graph.load();
+                await syncAllTsconfigs(graph);
+            }
+            break;
+    }
+}

package/src/building/error.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export class InternalBuildError extends Error {
+    constructor(message: string) {
+        super(message);
+    }
+}
+
+export class BuildError extends Error {
+    constructor(readonly diagnostics: string) {
+        super();
+    }
+
+    override get stack() {
+        return this.diagnostics;
+    }
+
+    override toString() {
+        return this.diagnostics;
+    }
+
+    inspect() {
+        return this.toString();
+    }
+}

package/src/building/graph.ts

@@ -0,0 +1,251 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import mapWorkspaces from "@npmcli/map-workspaces";
+import colors from "ansi-colors";
+import { JsonNotFoundError, Package } from "../util/package.js";
+import { Progress } from "../util/progress.js";
+import { Builder } from "./builder.js";
+import { InternalBuildError } from "./error.js";
+import { BUILD_INFO_LOCATION, BuildInformation, Project } from "./project.js";
+
+/**
+ * Graph of dependencies for workspace packages.
+ *
+ * We use this information to determine which packages are "dirty" and need rebuild.  In the future we can also use for
+ * parallel build, only tricky part there is showing status.
+ */
+export class Graph {
+    protected constructor(readonly nodes: Graph.Node[]) {}
+
+    get(name: string) {
+        const node = this.nodes.find(node => node.pkg.name === name);
+        if (node === undefined) {
+            throw new Error(`Cannot locate package "${name}"`);
+        }
+        return node;
+    }
+
+    static async load(pkg = Package.workspace) {
+        const nodeMap = await this.#loadNodes(pkg.workspace);
+        return await this.#createGraph(Object.values(nodeMap));
+    }
+
+    static async forProject(path: string): Promise<Graph | undefined> {
+        let workspace;
+        try {
+            workspace = Package.workspaceFor(path);
+        } catch (e) {
+            if (e instanceof JsonNotFoundError) {
+                // Project is not in a workspace
+                return;
+            }
+            throw e;
+        }
+
+        const nodeMap = await this.#loadNodes(workspace);
+
+        const rootPkg = new Package({ path: path });
+        const rootNode = nodeMap[rootPkg.json.name];
+        if (!rootNode) {
+            // Project resides under a workspace but is not part of the workspace
+            return;
+        }
+
+        const nodes = new Set<Graph.Node>();
+        function addNode(node: Graph.Node) {
+            if (nodes.has(node)) {
+                return;
+            }
+            nodes.add(node);
+            for (const dependency of node.dependencies) {
+                addNode(dependency);
+            }
+        }
+
+        addNode(rootNode);
+
+        return await this.#createGraph([...nodes]);
+    }
+
+    // TODO - parallelization will be trivial except need to update Progress
+    // to support display of multiple simultaneous tasks
+    async build(builder: Builder, showSkipped = true) {
+        const toBuild = new Set(this.nodes);
+
+        // We configure each build before building so that any generated files are in place before we initiate the build
+        for (const node of this.nodes) {
+            await builder.configure(node.project);
+        }
+
+        while (toBuild.size) {
+            let node;
+
+            nodes: for (node of toBuild) {
+                for (const dep of node.dependencies) {
+                    if (dep.isDirty) {
+                        continue nodes;
+                    }
+                }
+                break;
+            }
+
+            if (!node) {
+                throw new Error("Internal logic error: No unbuilt project has fully built dependencies");
+            }
+
+            if (node.isDirty || builder.unconditional) {
+                await builder.build(node.project);
+                node.info.timestamp = new Date().toISOString();
+            } else if (showSkipped) {
+                new Progress().skip("Up to date", node.pkg);
+            }
+
+            toBuild.delete(node);
+        }
+    }
+
+    display() {
+        for (const node of this.nodes) {
+            const progress = node.pkg.start("Node");
+            progress.info("path", node.pkg.path);
+            progress.info("modified", formatTime(node.modifyTime));
+            progress.info("built", formatTime(node.info.timestamp ?? 0));
+            progress.info("dirty", node.isDirty ? colors.dim.red("yes") : colors.dim.green("no"));
+            progress.info("dependencies", node.dependencies.map(formatDep).join(", "));
+            progress.shutdown();
+        }
+    }
+
+    static async #createGraph(nodes: Graph.Node[]) {
+        const graph = new Graph(nodes);
+
+        await Promise.all(
+            graph.nodes.map(async node => {
+                if (node.pkg.hasFile(BUILD_INFO_LOCATION)) {
+                    node.info = await node.pkg.readJson(BUILD_INFO_LOCATION);
+                }
+
+                node.modifyTime = await node.pkg.lastModified("package.json", "src", "test");
+
+                return node;
+            }),
+        );
+
+        const stack = Array<Graph.Node>();
+        function findCircular(node: Graph.Node) {
+            if (stack.indexOf(node) !== -1) {
+                stack.push(node);
+                throw new InternalBuildError(`Circular dependency: ${stack.map(formatDep).join(" ▸ ")}`);
+            }
+            stack.push(node);
+            for (const dep of node.dependencies) {
+                findCircular(dep);
+            }
+            stack.pop();
+        }
+        for (const node of graph.nodes) {
+            findCircular(node);
+        }
+
+        return graph;
+    }
+
+    static async #loadNodes(workspace: Package) {
+        const workspaces = await mapWorkspaces({ pkg: workspace.json, cwd: workspace.path });
+
+        const nodeMap = {} as Record<string, Graph.Node>;
+        const allDeps = {} as Record<string, string[]>;
+        for (const path of workspaces.values()) {
+            const pkg = new Package({ path: path });
+            allDeps[pkg.json.name] = pkg.dependencies;
+            nodeMap[pkg.json.name] = {
+                pkg,
+                project: new Project(pkg),
+                dependencies: [],
+                info: {},
+                modifyTime: 0,
+
+                get buildTime() {
+                    return this.info.timestamp ? new Date(this.info.timestamp).getTime() : 0;
+                },
+
+                get isDirty() {
+                    return (
+                        this.modifyTime > this.buildTime ||
+                        !!this.dependencies.find(
+                            dep =>
+                                dep.isDirty ||
+                                (dep.buildTime > this.buildTime &&
+                                    (dep.info.apiSha === undefined ||
+                                        dep.info.apiSha !== this.info.dependencyApiShas?.[dep.pkg.name])),
+                        )
+                    );
+                },
+
+                // This version of dirty should only be used after building all dependencies.  If this project is
+                // unmodified and the SHA of the type definitions of the target project is also unmodified then we do
+                // not need to build
+                get prebuildDirty() {
+                    if (this.modifyTime > this.buildTime) {
+                        return true;
+                    }
+                    const depShas = this.info.dependencyApiShas;
+                    if (typeof depShas !== "object") {
+                        return this.isDirty;
+                    }
+                    for (const dep of this.dependencies) {
+                        const depSha = depShas[dep.pkg.name];
+                        if (depSha === undefined || depSha !== dep.info.apiSha) {
+                            return true;
+                        }
+                    }
+                    return false;
+                },
+            };
+        }
+
+        for (const name in allDeps) {
+            for (const dep of allDeps[name]) {
+                const depNode = nodeMap[dep];
+
+                // Note -- allow nodes to reference themselves, seems to be necessary on tools for use of tsc
+                if (depNode && depNode !== nodeMap[name]) {
+                    nodeMap[name].dependencies.push(depNode);
+                }
+            }
+        }
+
+        return nodeMap;
+    }
+}
+
+export namespace Graph {
+    export interface Node {
+        pkg: Package;
+        project: Project;
+        dependencies: Node[];
+        buildTime: number;
+        info: BuildInformation;
+        modifyTime: number;
+        isDirty: boolean;
+        prebuildDirty: boolean;
+    }
+}
+
+function formatTime(time: number | string) {
+    if (!time) {
+        return colors.dim.red("never");
+    }
+    if (typeof time === "string") {
+        time = new Date(time).getTime();
+    }
+    return new Date(time - new Date().getTimezoneOffset()).toISOString().split(".")[0].replace("T", " ");
+}
+
+function formatDep(node: Graph.Node) {
+    return node.pkg.name.replace(/^@(?:project-chip|matter\.js)\//, "");
+}

package/src/building/project.ts

@@ -0,0 +1,313 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { Hash } from "crypto";
+import { build as esbuild, Format } from "esbuild";
+import { cp, mkdir, readFile, rm, stat, symlink, writeFile } from "fs/promises";
+import { glob } from "glob";
+import { platform } from "os";
+import { dirname, join, relative } from "path";
+import { ignoreError } from "../util/errors.js";
+import { CONFIG_PATH, Package } from "../util/package.js";
+import { Progress } from "../util/progress.js";
+
+export const BUILD_INFO_LOCATION = "build/info.json";
+
+export interface BuildInformation {
+    /**
+     * Time of last build.  Compared to source files to determine whether build is dirty.
+     */
+    timestamp?: string;
+
+    /**
+     * API signature.  Used by dependents to determine whether they need to rebuild after we do.
+     */
+    apiSha?: string;
+
+    /**
+     * API signature of each dependency.  Compared to apiSha of each dependency during dirty detection.
+     */
+    dependencyApiShas?: Record<string, string>;
+}
+
+export class Project {
+    pkg: Package;
+    #config?: Project.Config;
+    #configured?: boolean;
+
+    constructor(source: Package | string = ".") {
+        if (typeof source === "string") {
+            this.pkg = new Package({ path: source });
+        } else {
+            this.pkg = source;
+        }
+
+        if (!this.pkg.hasSrc) {
+            throw new Error(`Found package ${this.pkg.json.name} but no src directory is present`);
+        }
+    }
+
+    async buildSource(format: Format) {
+        await this.#build(format, "src", `dist/${format}`);
+        await this.#configureFormat("dist", format, true);
+    }
+
+    async buildTests(format: Format) {
+        if (this.pkg.hasDirectory("test")) {
+            await this.#build(format, "test", `build/${format}/test`);
+        }
+
+        const src = `dist/${format}`;
+        const dest = `build/${format}/src`;
+
+        try {
+            const destPath = this.pkg.resolve(dest);
+            await mkdir(dirname(destPath), { recursive: true });
+            await ignoreError("EEXIST", async () => await symlink(this.pkg.resolve(src), destPath));
+        } catch (e) {
+            if ((e as any).code === "EPERM" && platform() === "win32") {
+                // If developer mode is not enabled, we can't create a symlink
+                // on Windows.  Copy instead
+                await cp(this.pkg.resolve(src), this.pkg.resolve(dest), {
+                    recursive: true,
+                    force: true,
+                });
+            } else {
+                throw e;
+            }
+        }
+        await this.#configureFormat("build", format, false);
+    }
+
+    async clean() {
+        for (const dir of ["build", "dist"]) {
+            await rm(this.pkg.resolve(dir), { recursive: true, force: true });
+        }
+    }
+
+    /**
+     * Installs declaration files into specified directory.  Also computes SHA-1 hash for all files that we use to
+     * detect "public API changes".
+     *
+     * Note we use cp to traverse filesystem, detect changes, locate source maps and compute API SHA.  A little
+     * convoluted but works.  When you correct for a node 20/windows bug.
+     */
+    async installDeclarationFormats(formats: Iterable<string>, apiSha?: Hash) {
+        const srcMaps = Array<string>();
+        let firstPass = true;
+
+        const src = this.pkg.resolve("build/types/src");
+
+        for (const format of formats) {
+            await cp(src, this.pkg.resolve(`dist/${format}`), {
+                recursive: true,
+                force: true,
+
+                filter: async (source, destination) => {
+                    const sourceStat = await stat(source);
+                    if (!sourceStat.isFile()) {
+                        return true;
+                    }
+
+                    // Ignore files that are unchanged
+                    const destinationMtime = (await ignoreError("ENOENT", () => stat(destination)))?.mtimeMs;
+                    if (destinationMtime !== undefined) {
+                        const sourceMtime = sourceStat.mtimeMs;
+                        if (destinationMtime >= sourceMtime) {
+                            return false;
+                        }
+                    }
+
+                    // We process source maps below
+                    if (source.endsWith(".d.ts.map")) {
+                        if (firstPass) {
+                            if (source.startsWith("\\\\?\\")) {
+                                // Node 20 prefixes the path with above on Windows and relative() can't handle it;
+                                // just strip off
+                                source = source.slice(4);
+                            }
+                            srcMaps.push(source);
+                        }
+                        return false;
+                    }
+
+                    // Update hash if provided
+                    if (apiSha) {
+                        apiSha.update(await readFile(source));
+                    }
+
+                    return true;
+                },
+            });
+
+            // Only need to collect source maps and update hash on first pass
+            firstPass = false;
+        }
+
+        // If you specify --sourceRoot, tsc just sticks whatever the string is directly into the file.  Not very useful
+        // unless you have no hierarchy or use absolute paths...
+        //
+        // We distribute types for src one level higher than we generate them (dist/esm vs build/types/src) so the paths
+        // end up incorrect.
+        //
+        // So...  Rewrite the paths in all source maps under src/.  Do this directly on buffer for marginal performance
+        // win.
+        for (const filename of srcMaps) {
+            // Load map as binary
+            let map = await readFile(filename);
+
+            // Find key text
+            let pos = map.indexOf('"sources":["../');
+            if (pos === -1) {
+                throw new Error(
+                    `Could not find sources position in declaration map ${filename}, format may have changed`,
+                );
+            }
+
+            // move to ../
+            pos += 12;
+
+            // Shift everything left by three
+            map = map.copyWithin(pos, pos + 3).subarray(0, map.length - 3);
+
+            // Write to new locations
+            const pathRelativeToDest = relative(src, filename);
+            for (const format of formats) {
+                await writeFile(join(join(this.pkg.resolve(`dist/${format}`), pathRelativeToDest)), map);
+            }
+        }
+    }
+
+    get hasDeclarations() {
+        return this.pkg.hasDirectory("build/types");
+    }
+
+    async installDeclarations(apiSha?: Hash) {
+        await mkdir(this.pkg.resolve("dist"), { recursive: true });
+        const formats = new Set<string>();
+        if (this.pkg.supportsEsm) {
+            formats.add("esm");
+        }
+        if (this.pkg.supportsCjs) {
+            formats.add("cjs");
+        }
+        await this.installDeclarationFormats(formats, apiSha);
+    }
+
+    async recordBuildInfo(info: BuildInformation) {
+        await mkdir(this.pkg.resolve("build"), { recursive: true });
+        info.timestamp = new Date().toISOString();
+        await writeFile(this.pkg.resolve(BUILD_INFO_LOCATION), JSON.stringify(info, undefined, 4));
+    }
+
+    async configure(progress: Progress) {
+        if (this.#configured) {
+            return this.#config;
+        }
+
+        if (!this.pkg.hasConfig) {
+            return {};
+        }
+
+        const configPath = this.pkg.resolve(CONFIG_PATH);
+
+        const outfile = configPath.replace(/\\/g, "/").replace("/src/", "/build/").replace(/\.ts$/, ".js");
+        await esbuild({
+            entryPoints: [configPath],
+            outfile,
+            sourcemap: true,
+        });
+
+        this.#config = (await import(`file://${outfile}`)) as Project.Config | undefined;
+        this.#configured = true;
+
+        if (this.#config?.startup) {
+            await this.#config?.startup({
+                project: this,
+                progress,
+            });
+        }
+
+        return this.#config;
+    }
+
+    async #build(format: Format, indir: string, outdir: string) {
+        const entryPoints = await this.#targetsOf(indir, outdir, "ts", "js");
+
+        const configPath = this.pkg.resolve(CONFIG_PATH);
+        for (const entry of entryPoints) {
+            if (entry.in === configPath) {
+                continue;
+            }
+            entry.out = entry.out.replace(/\.[jt]s$/, "");
+        }
+
+        await esbuild({
+            entryPoints,
+            outdir: this.pkg.path,
+            format,
+            sourcemap: true,
+            sourcesContent: false,
+            absWorkingDir: dirname(this.pkg.resolve(indir)),
+
+            // This is necessary to downlevel "using"
+            target: "es2022",
+
+            logOverride: {
+                "direct-eval": "silent",
+            },
+        });
+
+        for (const entry of await this.#targetsOf(indir, outdir, "cjs", "mjs", "d.cts", "d.mts")) {
+            await cp(entry.in, entry.out);
+        }
+    }
+
+    async #configureFormat(dir: string, format: Format, isDist: boolean) {
+        // Build import map
+        let { imports } = this.pkg.json;
+        if (isDist && typeof imports === "object") {
+            imports = { ...imports };
+            for (const key in imports) {
+                const value = imports[key];
+                if (typeof value === "string") {
+                    imports[key] = value.replace(/^\.\/src\//, "./");
+                }
+            }
+        }
+
+        // Write package.json
+        const path = this.pkg.resolve(`${dir}/${format}/package.json`);
+        const json: Record<string, unknown> = {
+            type: format === "cjs" ? "commonjs" : "module",
+            imports,
+        };
+        await writeFile(path, JSON.stringify(json, undefined, 4));
+    }
+
+    async #targetsOf(indir: string, outdir: string, ...extensions: string[]) {
+        indir = this.pkg.resolve(indir).replace(/\\/g, "/");
+        outdir = this.pkg.resolve(outdir).replace(/\\/g, "/");
+
+        return (await glob(extensions.map(ext => `${indir}/**/*.${ext}`))).map(file => ({
+            in: file,
+            out: `${outdir}/${file.slice(indir.length + 1)}`,
+        }));
+    }
+}
+
+export namespace Project {
+    export interface Context {
+        project: Project;
+        progress: Progress;
+    }
+
+    export interface Config {
+        startup?: (context: Context) => Promise<void>;
+        before?: (context: Context) => Promise<void>;
+        after?: (context: Context) => Promise<void>;
+    }
+}