@matter/tools 0.11.0-alpha.0-20241005-e3e4e4a7a

package/src/building/tsconfig.ts

@@ -0,0 +1,69 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { relative, resolve } from "path";
+import { Graph } from "./graph.js";
+
+/**
+ * Unfortunately typescript's "project references" are redundant with package.json dependencies.  We don't use them for
+ * build but there are still some advantages to maintaining them so we just ensure they're in sync during workspace
+ * builds.
+ *
+ * One discussion on the topic: https://github.com/microsoft/TypeScript/issues/25376
+ */
+export async function syncAllTsconfigs(graph: Graph) {
+    const workspace = graph.nodes[0].pkg.workspace;
+    const rootTsconfig = await workspace.readJson("tsconfig.json");
+
+    const originalReferences = rootTsconfig.references;
+    rootTsconfig.references = [];
+
+    for (const node of graph.nodes) {
+        await syncPackageTsconfigs(graph, node);
+        rootTsconfig.references.push({ path: workspace.relative(node.pkg.path) });
+    }
+
+    if (referencesChanged(originalReferences, rootTsconfig.references)) {
+        await workspace.writeJson("tsconfig.json", rootTsconfig);
+    }
+}
+
+function referencesChanged(originalReferences: unknown, newReferences: unknown) {
+    return JSON.stringify(originalReferences) !== JSON.stringify(newReferences);
+}
+
+export async function syncPackageTsconfigs(graph: Graph, node: Graph.Node) {
+    await syncSubproject(node, "src");
+    await syncSubproject(node, "test", node.pkg.resolve("src"), graph.get("@matter/tools").pkg.resolve("src"));
+}
+
+async function syncSubproject(node: Graph.Node, path: string, ...extraRefs: string[]) {
+    path = node.pkg.resolve(path);
+
+    const tsconfigPath = resolve(path, "tsconfig.json");
+    if (!node.pkg.hasFile(tsconfigPath)) {
+        return;
+    }
+
+    const tsconfig = await node.pkg.readJson(tsconfigPath);
+
+    let refs = tsconfig.refs as undefined | { path: string }[];
+
+    if (refs === undefined) {
+        refs = [];
+    }
+
+    const deps = node.dependencies.map(dep => dep.pkg.resolve("src")).filter(p => !p.match(/packages\/tools/));
+
+    const desired = [...new Set([...deps, ...extraRefs])];
+
+    const newReferences = desired.map(ref => ({ path: relative(path, ref) }));
+
+    if (referencesChanged(tsconfig.references, newReferences)) {
+        tsconfig.references = newReferences;
+        await node.pkg.writeJson(tsconfigPath, tsconfig);
+    }
+}

package/src/building/typescript/context.ts

@@ -0,0 +1,85 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { dirname } from "path";
+import {
+    createCompilerHost,
+    Diagnostic,
+    formatDiagnosticsWithColorAndContext,
+    parseJsonConfigFileContent,
+    readConfigFile,
+    sys,
+} from "typescript";
+import { Package } from "../../util/package.js";
+import { BuildError } from "../error.js";
+
+export interface TypescriptContext {
+    build(pkg: Package, path: string, emit?: boolean): Promise<void>;
+}
+
+export namespace TypescriptContext {
+    export function diagnose(diagnostics?: Diagnostic | Diagnostic[]) {
+        if (diagnostics === undefined) {
+            return;
+        }
+        if (Array.isArray(diagnostics)) {
+            if (!diagnostics.length) {
+                return;
+            }
+        } else {
+            diagnostics = [diagnostics];
+        }
+
+        const host = createCompilerHost({});
+
+        let formatted = formatDiagnosticsWithColorAndContext(diagnostics, host);
+
+        // Strangely there are not newlines between errors in this output like there is when you run tsc from the
+        // command line.  Use the "light blue" ANSI escape code as an injection point for an additional newline
+        //
+        // eslint-disable-next-line no-control-regex
+        formatted = formatted.replace(/\u001b\[96m/gms, "\n\u001b[96m");
+
+        throw new BuildError(formatted);
+    }
+
+    export function compilerOptionsFor(path: string, ignoreErrors = false) {
+        const file = readConfigFile(path, sys.readFile);
+        diagnose(file.error);
+        const config = parseJsonConfigFileContent(file.config, sys, dirname(path));
+        if (!ignoreErrors) {
+            diagnose(config.errors);
+        }
+        return config.options;
+    }
+
+    export function instrumentHostForSpinner(host: {}, refreshCallback?: () => void) {
+        if (!refreshCallback) {
+            return;
+        }
+
+        // The refresh callback allows us to make spinner updates even though TS is synchronous.
+        //
+        // TODO - add interception points as spinner still hangs
+
+        for (const name of ["readFile", "writeFile", "getSourceFile"]) {
+            if (!(name in host)) {
+                continue;
+            }
+
+            const fn = (host as Record<string, () => unknown>)[name];
+            if (typeof fn !== "function") {
+                continue;
+            }
+
+            (host as Record<string, () => unknown>)[name] = (...args: any) => {
+                const result = fn.apply(host, args);
+                refreshCallback();
+                return result;
+            };
+        }
+    }
+}

package/src/building/typescript/incremental-compiler.ts

@@ -0,0 +1,126 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { existsSync } from "fs";
+import { CompilerOptions, createIncrementalCompilerHost, createIncrementalProgram } from "typescript";
+import { Package } from "../../util/package.js";
+import { InternalBuildError } from "../error.js";
+import { Graph } from "../graph.js";
+import { TypescriptContext } from "./context.js";
+
+export function createIncrementalCompilerContext(
+    _workspace: Package,
+    _graph: Graph | undefined,
+    refreshCallback?: () => void,
+): TypescriptContext {
+    const baseOptions = {
+        ...TypescriptContext.compilerOptionsFor(Package.tools.resolve("tsc/tsconfig.base.json")),
+
+        incremental: true,
+        isolatedModules: true,
+        skipLibCheck: true,
+    };
+
+    return { build };
+
+    async function build(pkg: Package, path: string, emit?: boolean) {
+        let options;
+        if (emit) {
+            options = {
+                outDir: pkg.resolve("build/types"),
+                emitDeclarationOnly: true,
+                sourceMap: true,
+                declarationMap: true,
+            };
+        } else {
+            options = {
+                noEmit: true,
+            };
+        }
+
+        options = {
+            ...baseOptions,
+            tsBuildInfoFile: pkg.resolve("build/tsbuildinfo"),
+            rootDir: pkg.path,
+            ...options,
+        };
+
+        delete options.composite;
+
+        if (pkg.hasSrc) {
+            loadConfiguredOptions(pkg.resolve("src/tsconfig.json"), options);
+        }
+
+        if (pkg.hasTests) {
+            loadConfiguredOptions(pkg.resolve("src/tsconfig.json"), options);
+        }
+
+        const host = createIncrementalCompilerHost(options);
+
+        TypescriptContext.instrumentHostForSpinner(host, refreshCallback);
+        const sources = Array<string>();
+
+        if (path === "src") {
+            if (pkg.hasSrc) {
+                sources.push(...(await pkg.glob("src/**/*.ts")));
+            }
+        } else if (path === "test") {
+            if (pkg.hasTests) {
+                sources.push(...(await pkg.glob("test/**/*.ts")));
+            }
+        } else {
+            throw new InternalBuildError(`Unsupported build path "${path}"`);
+        }
+
+        const program = createIncrementalProgram({
+            rootNames: sources,
+            options,
+            host,
+        });
+
+        // See https://github.com/microsoft/TypeScript/issues/31849
+        const diagnostics = [
+            ...program.getConfigFileParsingDiagnostics(),
+            ...program.getSyntacticDiagnostics(),
+            ...program.getOptionsDiagnostics(),
+            ...program.getSemanticDiagnostics(),
+        ];
+
+        if (!options.noEmit) {
+            diagnostics.push(...program.emit().diagnostics);
+        }
+
+        TypescriptContext.diagnose(diagnostics);
+    }
+
+    /**
+     * As we largely configure based on convention, we mostly ignore tsconfig.json files in project directories.  The
+     * limited number of project-specific options we allow load here.
+     */
+    function loadConfiguredOptions(path: string, into: CompilerOptions) {
+        if (!existsSync(path)) {
+            // Package tsconfigs are optional
+            return;
+        }
+
+        const options = TypescriptContext.compilerOptionsFor(path);
+
+        delete options?.composite;
+
+        const types = options?.types;
+        if (types) {
+            if (into.types) {
+                const merged = new Set(into.types);
+                for (const type of types) {
+                    merged.add(type);
+                }
+                into.types = [...merged];
+            } else {
+                into.types = types;
+            }
+        }
+    }
+}

package/src/building/typescript/language-service.ts

@@ -0,0 +1,132 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { statSync } from "fs";
+import { join } from "path";
+import {
+    createDocumentRegistry,
+    createLanguageService,
+    createModuleResolutionCache,
+    getDefaultLibFileName,
+    JSDocParsingMode,
+    LanguageServiceHost,
+    LanguageServiceMode,
+    ModuleKind,
+    ResolvedModuleWithFailedLookupLocations,
+    resolveModuleName,
+    ScriptSnapshot,
+} from "typescript";
+import { Package } from "../../util/package.js";
+import { Graph } from "../graph.js";
+import { TypescriptContext } from "./context.js";
+
+/**
+ * This is highly experimental and not yet fully functional.
+ */
+export async function createLanguageServiceContext(
+    workspace: Package,
+    _graph: Graph | undefined,
+    refreshCallback: () => void,
+): Promise<TypescriptContext> {
+    const baseOptions = TypescriptContext.compilerOptionsFor(Package.tools.resolve("tsc/tsconfig.base.json"), true);
+    const registry = createDocumentRegistry(true, workspace.path, JSDocParsingMode.ParseNone);
+    const resolutionCache = createModuleResolutionCache(workspace.path, name => name, baseOptions);
+    //const myPackageNames = new Set(graph?.nodes.map(node => node.pkg.name));
+
+    return { build };
+
+    async function build(pkg: Package, path: string, emit?: boolean) {
+        const files = await pkg.glob(join(path, "**/*.ts"));
+        const host = await createLanguageServiceHost(pkg, path, files);
+        const ls = createLanguageService(host, registry, LanguageServiceMode.Semantic);
+
+        TypescriptContext.diagnose(ls.getCompilerOptionsDiagnostics());
+
+        for (const path of files) {
+            TypescriptContext.diagnose([...ls.getSyntacticDiagnostics(path), ...ls.getSemanticDiagnostics(path)]);
+
+            if (emit) {
+                const relativePath = pkg.relative(path);
+                await pkg.writeFile(join("build/types", relativePath), ls.getEmitOutput(path));
+            }
+        }
+
+        TypescriptContext.diagnose(ls.getCompilerOptionsDiagnostics());
+    }
+
+    async function createLanguageServiceHost(pkg: Package, path: string, files: string[]) {
+        const host: LanguageServiceHost = {
+            getCompilationSettings() {
+                return TypescriptContext.compilerOptionsFor(join(path, "tsconfig.json"));
+            },
+
+            getScriptFileNames() {
+                return files;
+            },
+
+            getScriptVersion(filename) {
+                return statSync(filename).mtime.toISOString();
+            },
+
+            getScriptSnapshot(filename) {
+                try {
+                    return ScriptSnapshot.fromString(pkg.readFileSync(filename));
+                } catch (e) {
+                    switch ((e as { code: string }).code) {
+                        case "ENOENT":
+                        case "ENOTDIR":
+                            return;
+                    }
+                    throw e;
+                }
+            },
+
+            getCurrentDirectory() {
+                return path;
+            },
+
+            getDefaultLibFileName(options) {
+                return workspace.resolve(join("node_modules/typescript/lib", getDefaultLibFileName(options)));
+            },
+
+            readFile(path: string) {
+                refreshCallback();
+                return pkg.readFileSync(path);
+            },
+
+            fileExists(path: string) {
+                return pkg.hasFile(path);
+            },
+
+            resolveModuleNameLiterals(
+                moduleLiterals,
+                containingFile,
+                redirectedReferences,
+                options,
+                _containingSourceFile,
+                _reusedNames,
+            ): ResolvedModuleWithFailedLookupLocations[] {
+                return moduleLiterals.map(name => {
+                    const result = resolveModuleName(
+                        name.text,
+                        containingFile,
+                        options,
+                        host,
+                        resolutionCache,
+                        redirectedReferences,
+                        ModuleKind.ESNext,
+                    );
+                    console.log(name.text, "->", result.resolvedModule?.resolvedFileName);
+                    return result;
+                });
+            },
+
+            jsDocParsingMode: JSDocParsingMode.ParseNone,
+        };
+
+        return host;
+    }
+}

package/src/building/typescript/solution-builder.ts

@@ -0,0 +1,67 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { join } from "path";
+import { createSolutionBuilder, createSolutionBuilderHost, Diagnostic, ExitStatus } from "typescript";
+import { Package } from "../../util/package.js";
+import { BuildError, InternalBuildError } from "../error.js";
+import { Graph } from "../graph.js";
+import { TypescriptContext } from "./context.js";
+
+export async function createSolutionBuilderContext(
+    workspace: Package,
+    _graph: Graph | undefined,
+    refreshCallback?: () => void,
+): Promise<TypescriptContext> {
+    let diagnostics: undefined | Diagnostic[];
+    const host = createSolutionBuilderHost();
+    host.reportDiagnostic = diagnostic => (diagnostics ??= []).push(diagnostic);
+
+    const createProgram = host.createProgram;
+    host.createProgram = (...args) => {
+        const result = createProgram(...args);
+        return result;
+    };
+
+    const readDirectory = host.readDirectory;
+    host.readDirectory = (...args) => {
+        const result = readDirectory?.(...args);
+        return result ?? [];
+    };
+
+    const builder = createSolutionBuilder(host, [workspace.resolve("tsconfig.json")], {});
+
+    TypescriptContext.diagnose(diagnostics);
+
+    TypescriptContext.instrumentHostForSpinner(host, refreshCallback);
+
+    return {
+        async build(pkg: Package, path: string, _emit?: boolean) {
+            const status = builder.build(pkg.resolve(join(path, "tsconfig.json")));
+
+            TypescriptContext.diagnose(diagnostics);
+
+            switch (status) {
+                case undefined:
+                case ExitStatus.Success:
+                    break;
+
+                case ExitStatus.InvalidProject_OutputsSkipped:
+                    throw new BuildError(
+                        `Typescript does not think ${path} is part of the project, this is probably a build tooling error`,
+                    );
+
+                case ExitStatus.ProjectReferenceCycle_OutputsSkipped:
+                    throw new BuildError(
+                        `Typescript identified a cyclical project reference in ${path}, please check package dependencies`,
+                    );
+
+                default:
+                    throw new InternalBuildError(`Build error ${ExitStatus[status]} without diagnostics`);
+            }
+        },
+    };
+}

package/src/building/typescript.ts

@@ -0,0 +1,15 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { TypescriptContext } from "./typescript/context.js";
+import { createSolutionBuilderContext } from "./typescript/solution-builder.js";
+
+/**
+ * Create a {@link TypescriptContext} for validating types and optionally emitting .d.ts file.
+ *
+ * We have multiple implementations as we've tried TS API variants.  We export the one we currently use here.
+ */
+export const createTypescriptContext = createSolutionBuilderContext;

package/src/index.ts

@@ -0,0 +1,15 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import "./testing/global-declarations.js";
+export * from "./building/graph.js";
+export * from "./building/project.js";
+export * from "./testing/chip.js";
+export * from "./testing/mocharc.cjs";
+export * from "./util/docker.js";
+export * from "./util/package.js";
+export * from "./util/progress.js";
+export * from "./util/wtf.js";

package/src/running/cli.ts

@@ -0,0 +1,95 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { dirname, resolve } from "path";
+import { exit } from "process";
+import { Builder } from "../building/builder.js";
+import { Graph } from "../building/graph.js";
+import { Project } from "../building/project.js";
+import { executeNode } from "./execute.js";
+
+/**
+ * Execute a script in a package that conforms to matter.js conventions.  This is primarily useful within a matter.js
+ * development environment but should also work when installed via NPM.
+ *
+ * First we translate source filenames to the transpiled *.js equivalent.
+ *
+ * If the target script is in a package not installed via NPM we build the package.  If the package is an NPM workspace
+ * under a top-level package we also build dependencies.
+ *
+ * We then run the script in a child process so we can enable source maps using node --source-maps.  You can disable
+ * this by setting the environment variable MATTER_DIRECT_EXEC.
+ */
+export async function main(argv = process.argv) {
+    let script = argv[2];
+    argv = argv.slice(3);
+
+    if (script === undefined || script === "") {
+        console.error("Error: Script name required");
+        exit(1);
+    }
+
+    script = resolve(script);
+    let dir;
+
+    if (script.match(/[\\/]node_modules[\\/].bin[\\/]/)) {
+        // When executing a script linked under node_modules, search for the project from cwd.  This occurs when running
+        // tooling such as "matter-test"
+        dir = process.cwd();
+    } else {
+        // When executing outside of node modules we want to build the project containing the script
+        dir = dirname(script);
+    }
+
+    const project = new Project(dir);
+
+    let format: "esm" | "cjs";
+    if (project.pkg.supportsEsm) {
+        format = "esm";
+    } else if (project.pkg.supportsCjs) {
+        format = "cjs";
+    } else {
+        console.error("Error: Could not identify project format");
+        exit(2);
+    }
+
+    // In development we currently build package and dependencies unconditionally before running
+    const isDevelopment = !dir.match(/[\\/]node_modules[\\/]/);
+    if (isDevelopment) {
+        const builder = new Builder();
+        const dependencies = await Graph.forProject(dir);
+        if (dependencies) {
+            // Project is in a workspace; build along with dependencies from the same workspace
+            await dependencies.build(builder, false);
+        } else {
+            // Project is not in a workspace; only build the project
+            await builder.build(project);
+        }
+    }
+
+    // Determine the actual script to run
+    script = project.pkg.resolve(
+        project.pkg
+            .relative(script)
+            .replace(/\.ts$/, ".js")
+            .replace(/^src[\\/]/, `dist/${format}/`),
+    );
+
+    // If we run in the same process we cannot enable source maps so default mode is to fork.  However for development
+    // purposes it can be useful to avoid the intermediary process.  In this case you can set "--enable-source-maps"
+    // manually then set MATTER_DIRECT_EXEC
+    if (process.env.MATTER_DIRECT_EXEC) {
+        // This will not transpile properly to commonjs but we only use this module from ESM so that's OK
+        await import(script);
+    } else {
+        // Our behavior in response to SIGINT should mirror the child process's.  So ignore the signal locally, only
+        // quitting once the child process does
+        process.on("SIGINT", () => {});
+        process.on("SIGTERM", () => {});
+
+        process.exitCode = await executeNode(script, argv);
+    }
+}

package/src/running/execute.ts

@@ -0,0 +1,63 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { spawn, SpawnOptions } from "child_process";
+import { platform } from "os";
+
+import colors from "ansi-colors";
+
+export async function execute(bin: string, argv: string[], env?: typeof process.env) {
+    return new Promise<number>((resolve, reject) => {
+        let finished = false;
+
+        const options: SpawnOptions = {
+            stdio: "inherit",
+        };
+        if (platform() === "win32") {
+            options.shell = true;
+        }
+        if (env !== undefined) {
+            options.env = { ...process.env, ...env };
+        }
+
+        const proc = spawn(bin, argv, options);
+
+        // Proxy SIGUSR2 to the child process
+        process.on("SIGUSR2", () => {
+            proc.kill("SIGUSR2");
+        });
+
+        proc.on("error", e => {
+            finished = true;
+            reject(e);
+        });
+
+        proc.on("close", code => {
+            if (finished) {
+                return;
+            }
+            finished = true;
+
+            resolve(code ?? 1);
+        });
+    });
+}
+
+export async function executeNode(script: string, argv: string[]) {
+    argv = ["--enable-source-maps", script, ...argv];
+    if (process.env.MATTER_RUN_ECHO) {
+        const command = colors.whiteBright(`node ${argv.join(" ")}`);
+        process.stdout.write(`${colors.greenBright("Matter execute:")} ${command}\n`);
+    }
+    const env = {} as NodeJS.ProcessEnv;
+
+    // Hmm this is a little much as a default
+    // if (process.env.MATTER_LOG_STACK_LIMIT === undefined) {
+    //     env.MATTER_LOG_STACK_LIMIT = "100";
+    // }
+
+    return execute("node", argv, env);
+}