@matter/tools 0.11.0-alpha.0-20241005-e3e4e4a7a

package/src/testing/chip/pics-file.ts

@@ -0,0 +1,71 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { readFileSync, writeFileSync } from "fs";
+
+function readValue(line: string) {
+    const valueMatch = line.match(/^(\S+)=(.*)$/);
+    if (valueMatch) {
+        return { key: valueMatch[1], value: valueMatch[2] };
+    }
+    return { key: undefined, value: undefined };
+}
+
+/**
+ * Manages Matter PICS files.
+ *
+ * Note that we sometimes use ".properties" extension for PICS files so we get syntax highlighting, but PICS only
+ * supports a subset of the actual Java properties file format.
+ */
+export class PicsFile {
+    #lines: string[];
+    #values?: Record<string, string>;
+
+    constructor(pathOrBody: string, inline = false) {
+        if (inline === false) {
+            pathOrBody = readFileSync(pathOrBody, "utf-8");
+        }
+        this.#lines = pathOrBody.split("\n").map(l => l.trim());
+    }
+
+    get lines() {
+        return this.#lines;
+    }
+
+    get values() {
+        if (!this.#values) {
+            const values = {} as Record<string, string>;
+            for (const line of this.lines) {
+                const { key, value } = readValue(line);
+                if (key !== undefined) {
+                    values[key] = value;
+                }
+            }
+            this.#values = values;
+        }
+        return this.#values;
+    }
+
+    patch(other: PicsFile) {
+        this.#lines = this.lines.map(line => {
+            const { key } = readValue(line);
+            if (key === undefined) {
+                return line;
+            }
+
+            const otherValue = other.values[key];
+            if (otherValue === undefined) {
+                return line;
+            }
+
+            return `${key}=${otherValue}`;
+        });
+    }
+
+    save(path: string) {
+        writeFileSync(path, this.lines.join("\n"));
+    }
+}

package/src/testing/chip/pics.properties

@@ -0,0 +1,53 @@
+# @license
+# Copyright 2022-2024 Matter.js Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# These are overrides for CHIP "PICS" values.  We use values from CHIP's "ci-pics-values" with changes listed here.
+
+# This is CHIP's file format.  It is not (intentionally) a Java .properties file 🙄 but it is a subset and we get syntax
+# highlighting with the "properties" extension.
+
+# We support the eventlist global attribute and so also events
+PICS_EVENT_LIST_ENABLED=1
+
+# We support "absolute position feature" and want to run tests for that too
+WNCV.S.F03=1
+
+# We need to turn off calibration because the test expect a different behavior then we do in default implementation
+WNCV.S.M.Calibration=0
+
+# We support "Hue/Saturation" feature and want to run tests for that too
+CC.S.F00=1
+
+# We do not add attributes 3..7 to our test
+DGGEN.S.A0003=0
+DGGEN.S.A0004=0
+DGGEN.S.A0005=0
+DGGEN.S.A0006=0
+DGGEN.S.A0007=0
+
+# We do not support the "TestTrigger" command
+| DGGEN.S.C00.Rsp=0
+
+# We do not support the "TimeSnapshotResponse" command
+DGGEN.S.C03.Rsp=0
+
+# We do not enable events 0..2 for now
+DGGEN.S.E00=0
+DGGEN.S.E01=0
+DGGEN.S.E02=0
+
+# We do not support the "DM Test" feature
+DGGEN.S.F00=0
+
+# We do not support the optional Battery Fault Change event
+PS.S.E01=0
+
+# We use an Ethernet Network commissioning cluster and so we do not have these two attributes
+CNET.S.A0002..3=0
+
+# We do not provide a Taglist on Descriptor cluster
+DESC.S.F00=0
+
+# We provide a Constant pressure pump, so these attributes must not exist
+PCC.S.A0005..c=0

package/src/testing/chip.ts

@@ -0,0 +1,420 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { parse } from "path";
+import { Docker } from "../util/docker.js";
+import { Package } from "../util/package.js";
+import { PicsFile } from "./chip/pics-file.js";
+import { type TestRunner } from "./runner.js";
+
+/**
+ * Path configuration.
+ */
+namespace Constants {
+    export const chip = "/connectedhomeip";
+    export const chipTool = `${chip}/scripts/tests/chipyaml/chiptool.py`;
+    export const yamlTests = `${chip}/src/app/tests/suites/certification`;
+    export const pythonTests = `${chip}/src/python_testing`;
+    export const python = ["/usr/bin/env", "-S", "python3", "-B"];
+    export const pythonRunner = `${chip}/scripts/tests/run_python_test.py`;
+
+    export const pics = "/matter.js/packages/tools/build/pics.properties";
+    export const chipPics = "/connectedhomeip/src/app/tests/suites/certification/ci-pics-values";
+    export const buildTimeout = 600_000;
+    export const defaultTimeout = 60_000;
+    export const dockerBuildPath = "chip";
+    export const dockerName = "matterjs-chip";
+}
+
+/**
+ * Internal state.
+ */
+const State = {
+    configured: false,
+    options: undefined as Chip.Options | undefined,
+    docker: undefined as Docker | undefined,
+    yamlTests: Array<string>(),
+    pythonTests: Array<string>(),
+};
+
+/**
+ * Internal configuration management.
+ */
+const Config = {
+    set options(options: Chip.Options) {
+        State.options = options;
+    },
+
+    get runner() {
+        const runner = State.options?.runner;
+
+        if (runner === undefined) {
+            throw new Error("No test runner configured");
+        }
+
+        return runner;
+    },
+
+    async docker() {
+        if (State.docker) {
+            return State.docker;
+        }
+
+        const docker = new Docker();
+
+        const { progress } = Config.runner;
+
+        await progress.run(`Build ${progress.emphasize("CHIP image")}`, () =>
+            docker.buildImage(Constants.dockerName, Package.tools.resolve(Constants.dockerBuildPath)),
+        );
+
+        State.docker = docker;
+
+        return docker;
+    },
+};
+
+/**
+ * CHIP testing controller.  "CHIP tests" are official tests implemented in the connectedhomeip repository.
+ */
+export const Chip = {
+    /**
+     * Configure CHIP testing.  Invoke prior to use of other methods.
+     */
+    set config(config: Chip.Options) {
+        Config.options = config;
+    },
+
+    /**
+     * Initialize.  This must run before defining tests to enable test definition via globs.
+     */
+    async initialize() {
+        await initialize();
+    },
+
+    /**
+     * Define YAML tests.  This is a declarative CHIP test defined in a YAML file.
+     */
+    yaml(testee: Chip.Testee, includeGlob: string, excludeGlob?: string) {
+        let tests = filterWithGlob(State.yamlTests, includeGlob);
+        if (excludeGlob !== undefined) {
+            tests = filterWithGlob(tests, excludeGlob, true);
+        }
+
+        if (tests.length === 0) {
+            throw new Error(`YAML test glob ${includeGlob} matched no tests`);
+        }
+
+        for (const file of tests) {
+            implementTest(testee, {
+                name: parse(file).base,
+
+                command: [Constants.pythonRunner, file],
+
+                // TODO - complete argument set
+                args: {
+                    "--pics-file": Constants.pics,
+                    "--discriminator": "1234",
+                    "--passcode": "20202021",
+                },
+            });
+        }
+    },
+
+    /**
+     * Define a "python" test.  This is a CHIP test implemented as a python script.
+     */
+    python(testee: Chip.Testee, tester: Chip.TestSelection, excludeGlob?: string) {
+        if (typeof tester === "string") {
+            tester = {
+                name: tester,
+            };
+        } else {
+            tester = { ...tester };
+        }
+
+        // TODO - complete arguments
+        tester.args = {
+            "--commissioning-method": "on-network",
+            "--PICS": Constants.pics,
+            "--discriminator": "1234",
+            "--passcode": "20202021",
+            ...tester.args,
+        };
+
+        let files = filterWithGlob(State.pythonTests, tester.name, false);
+        if (excludeGlob !== undefined) {
+            files = filterWithGlob(files, excludeGlob, true);
+        }
+
+        if (files.length === 0) {
+            throw new Error(`Python test glob ${tester.name} matched no tests`);
+        }
+
+        for (const file of files) {
+            const name = parse(file).name;
+            implementTest(testee, { ...tester, name, command: [file] });
+        }
+    },
+};
+
+let containerInitializerInstalled = false;
+
+function implementTest(testee: Chip.Testee, tester: Chip.Tester) {
+    if (!containerInitializerInstalled) {
+        containerInitializerInstalled = true;
+        before(async function () {
+            this.timeout(Constants.buildTimeout);
+            await initialize();
+        });
+    }
+
+    it(tester.description ?? tester.name, async () => {
+        await testee.setup();
+        await testee.start();
+
+        try {
+            await invokeTester(tester);
+        } finally {
+            try {
+                await testee.stop();
+            } catch (e) {
+                console.warn("Error stopping test subject", e);
+            }
+        }
+    }).timeout(tester.timeout ?? Constants.defaultTimeout);
+}
+
+async function invokeTester(tester: Chip.Tester) {
+    const docker = await Config.docker();
+
+    const args = Array<string>();
+    if (tester.command) {
+        args.push(...tester.command);
+    }
+    if (tester.args) {
+        args.push(...Object.entries(tester.args).flat());
+    }
+
+    // TODO - define docker network to match CHIP's testing infrastructure
+    const output = docker.run(Constants.dockerName, {
+        args,
+        env: tester.environment,
+        binds: {
+            [Package.workspace.path]: "/matter.js",
+        },
+        network: "host",
+    });
+
+    await translateTestOutput(Config.runner, output);
+}
+
+export namespace Chip {
+    /**
+     * The test subject.
+     */
+    export interface Testee {
+        setup(): Promise<void>;
+        start(): Promise<void>;
+        stop(): Promise<void>;
+    }
+
+    /**
+     * The test implementation.
+     */
+    export type TestSelection = Tester | string;
+
+    /**
+     * Configuration required from testing program.
+     */
+    export interface Options {
+        runner: TestRunner;
+    }
+
+    /**
+     * Details of how to run a specific test.
+     */
+    export interface Tester {
+        name: string;
+        description?: string;
+        command?: string[];
+        args?: Record<string, string>;
+        timeout?: number;
+        environment?: Record<string, string>;
+    }
+}
+
+export type Chip = typeof Chip;
+
+/**
+ * Strip ANSI escape codes from a string.
+ */
+function deansify(text: string) {
+    // Credit to https://stackoverflow.com/questions/25245716/remove-all-ansi-colors-styles-from-strings
+    // eslint-disable-next-line no-control-regex
+    return text.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, "");
+}
+
+// TODO - can simplify because we've removed two layers of CHIP extraction.
+//
+// TODO - Adaptation to support both python and yaml tests
+async function translateTestOutput(runner: TestRunner, output: AsyncIterable<string>) {
+    let testCount = 0;
+
+    let capture = Array<string>();
+
+    let partial: string | undefined;
+
+    /**
+     * Extract failure details from the test runner's output sphaghetti.
+     */
+    function reportFailures() {
+        let testName: undefined | string;
+        let testLogs: undefined | string[];
+        for (const line of capture) {
+            const subtestBoundary = deansify(line).match(/\*{5} (Test Step \d+|Test Failure) : (.*)/);
+            if (subtestBoundary) {
+                if (subtestBoundary[1] === "Test Failure") {
+                    if (testName === undefined) {
+                        continue;
+                    }
+                    runner.reporter.failTest(testName, {
+                        message: subtestBoundary[2],
+                        logs: testLogs?.join("\n"),
+                    });
+                } else {
+                    testName = subtestBoundary[2];
+                    testLogs = [];
+                }
+            } else if (testLogs) {
+                // Extract matter.js logs
+                const appOut = line.match(/^.* APP {2}OUT {2}: (.*)$/);
+                if (appOut) {
+                    testLogs.push(appOut[1]);
+                    continue;
+                }
+
+                // Extract diagnostics from the test framework
+                const testOut = line.match(/(\d{4}-\d\d-\d\d) .*(\d\d:\d\d:\d\d\.\d\d\d) - TEST OUT {2}: (.*)$/);
+                if (testOut) {
+                    if (deansify(testOut[3]).match(/^(?:✗ \d+.\d+ms$|\s+\d+\. Running )/)) {
+                        continue;
+                    }
+
+                    const date = testOut[1];
+                    const time = testOut[2];
+                    const message = testOut[3].replace("\t\t    ", "").replace(/\t/g, "  ");
+
+                    // Test suite logs date and time of report plus time of log message.  We only care about the second
+                    // time so ignore the first
+                    testLogs.push(`CHIP ${date} ${time} ${message}`);
+                    continue;
+                }
+
+                // We don't recognize the line; pass as-is
+                testLogs.push(line);
+            }
+        }
+    }
+
+    /**
+     * Process test suite output.
+     *
+     * Unfortunately run_test_suite.py hides most output unless the test crashes so we can't count subtests reliably
+     * without PR or replacing with our own thing.  The individual test runners do report this though so we can report
+     * per-test information from failure output.
+     */
+    function processLine(line: string) {
+        // Inject the message into our log stream
+        MockLogger.injectExternalMessage("CHIP", line);
+
+        line = line.trim();
+        const plain = deansify(line);
+        const testBoundary = plain.match(
+            /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3} [A-Z ]{7} ([a-zA-Z0-9_]+)\s+- (Starting test|(?:Completed|FAILED) in \d+\.\d+ seconds)$/,
+        );
+
+        if (testBoundary) {
+            if (testBoundary[2].startsWith("Start")) {
+                testCount++;
+                capture = [];
+                runner.reporter.beginRun(testBoundary[1], undefined, false);
+            } else if (testBoundary[2].startsWith("Completed")) {
+                runner.reporter.endRun();
+            } else {
+                reportFailures();
+                runner.reporter.endRun();
+            }
+        }
+
+        capture.push(line);
+    }
+
+    for await (const chunk of output) {
+        const lines = chunk.split("\n");
+        if (partial) {
+            lines[0] = partial + lines[0];
+        }
+
+        partial = lines.pop();
+
+        for (const line of lines) {
+            processLine(line);
+        }
+    }
+
+    if (partial !== undefined) {
+        processLine(partial);
+    }
+}
+
+async function initialize() {
+    if (State.configured) {
+        return;
+    }
+
+    await Config.docker();
+    await configurePics();
+    await configureYaml();
+    await configurePython();
+
+    State.configured = true;
+}
+
+async function configurePics() {
+    const docker = await Config.docker();
+    const ciPics = await docker.readFileFromImage(Constants.dockerName, Constants.chipPics);
+    const pics = new PicsFile(ciPics, true);
+
+    const overrides = new PicsFile(Package.tools.resolve("src/testing/chip/pics.properties"));
+    pics.patch(overrides);
+
+    pics.save(Package.tools.resolve("build/pics.properties"));
+}
+
+async function configureYaml() {
+    const docker = await Config.docker();
+
+    const tests = await docker.resolveGlobFromImage(Constants.dockerName, `${Constants.yamlTests}/Test_*.yaml`);
+
+    State.yamlTests.push(...tests);
+}
+
+async function configurePython() {
+    const docker = await Config.docker();
+
+    const tests = (await docker.resolveGlobFromImage(Constants.dockerName, `${Constants.pythonTests}/*.py`)).filter(
+        name => name.match(/(?:TC_|Test)[^/]\.py$/),
+    );
+
+    State.pythonTests.push(...tests);
+}
+
+function filterWithGlob(list: string[], glob: string, invert = false) {
+    const globPattern = glob.replace(/\*/g, "[^\\/]+");
+    const pattern = new RegExp(`^.*/(?:Test_TC_|TC|Test)${globPattern}\\.(?:py|yaml)$`);
+    return list.filter(s => !!s.match(pattern) === !invert);
+}

package/src/testing/cli.ts

@@ -0,0 +1,122 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import "../util/node-shims.js";
+import "./global-definitions.js";
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { Builder } from "../building/builder.js";
+import { Graph } from "../building/graph.js";
+import { Project } from "../building/project.js";
+import { TestRunner } from "./runner.js";
+
+enum TestType {
+    esm = "esm",
+    cjs = "cjs",
+    web = "web",
+}
+
+Error.stackTraceLimit = 50;
+
+export async function main(argv = process.argv) {
+    const testTypes = new Set<TestType>();
+
+    let manual = false;
+
+    const args = await yargs(hideBin(argv))
+        .usage("Runs tests in packages adhering to matter.js standards.")
+        .option("prefix", {
+            alias: "p",
+            default: ".",
+            type: "string",
+            describe: "specify directory of package to test",
+        })
+        .option("web", {
+            alias: "w",
+            default: false,
+            type: "boolean",
+            describe: "enable web tests in default test mode",
+        })
+        .option("spec", {
+            type: "array",
+            string: true,
+            describe: "One or more paths of tests to run",
+            default: "./test/**/*Test.ts",
+        })
+        .option("all-logs", { type: "boolean", describe: "Emit log messages in real time" })
+        .option("debug", { type: "boolean", describe: "Enable Mocha debugging" })
+        .option("environment", { alias: "e", type: "string", describe: "Select named test environment" })
+        .option("fgrep", { alias: "f", type: "string", describe: "Only run tests matching this string" })
+        .option("force-exit", { type: "boolean", describe: "Force Node to exit after tests complete" })
+        .option("grep", { alias: "g", type: "string", describe: "Only run tests matching this regexp" })
+        .option("invert", { alias: "i", type: "boolean", describe: "Inverts --grep and --fgrep matches" })
+        .option("profile", { type: "boolean", describe: "Write profiling data to build/profiles (node only)" })
+        .option("wtf", { type: "boolean", describe: "Enlist wtfnode to detect test leaks" })
+        .option("trace-unhandled", { type: "boolean", describe: "Detail unhandled rejections with trace-unhandled" })
+        .command("*", "run all supported test types")
+        .command("esm", "run tests on node (ES6 modules)", () => testTypes.add(TestType.esm))
+        .command("cjs", "run tests on node (CommonJS modules)", () => testTypes.add(TestType.cjs))
+        .command("web", "run tests in web browser", () => testTypes.add(TestType.web))
+        .command("manual", "start test server and print URL for manual testing", () => {
+            testTypes.add(TestType.web);
+            manual = true;
+        })
+        .strict().argv;
+
+    // If spec specified and prefix is default, use the spec file to locate the package
+    let packageLocation = args.prefix;
+    if (packageLocation === "." && args.spec) {
+        const firstSpec = Array.isArray(args.spec) ? args.spec[0] : args.spec;
+        packageLocation = firstSpec;
+    }
+
+    const project = new Project(packageLocation);
+
+    process.chdir(project.pkg.path);
+
+    // If no test types are specified explicitly, run all enabled types
+    if (!testTypes.size) {
+        if (project.pkg.supportsEsm) {
+            testTypes.add(TestType.esm);
+        }
+        if (project.pkg.supportsCjs) {
+            testTypes.add(TestType.cjs);
+        }
+        if (args.web) {
+            testTypes.add(TestType.web);
+        }
+    }
+
+    const builder = new Builder();
+    const dependencies = await Graph.forProject(packageLocation);
+    if (dependencies) {
+        await dependencies.build(builder, false);
+    } else {
+        await builder.build(project);
+    }
+
+    const progress = project.pkg.start("Testing");
+    const runner = new TestRunner(project.pkg, progress, args);
+
+    if (testTypes.has(TestType.esm)) {
+        await runner.runNode("esm");
+    }
+
+    if (testTypes.has(TestType.cjs)) {
+        await runner.runNode("cjs");
+    }
+
+    if (testTypes.has(TestType.web)) {
+        await runner.runWeb(manual);
+    }
+
+    progress.shutdown();
+
+    if (args.forceExit) {
+        process.exit(0);
+    }
+}