@iamsergio/qttest-utils 2.2.1 → 2.3.0

package/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "2.3.0"
+}

package/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## [2.3.0](https://github.com/KDAB/qttest-utils/compare/v2.2.2...v2.3.0) (2024-06-06)
+
+
+### Features
+
+* Add support for QEXPECT_FAIL ([0037921](https://github.com/KDAB/qttest-utils/commit/003792112bd6093640e772dcfd109812f38324bd))
+* Add support for XPASS ([a60be6b](https://github.com/KDAB/qttest-utils/commit/a60be6b81f22d3a18ee624e2414e91c37e2c607f))
+
+
+### Bug Fixes
+
+* Fix JSON output from ctest not being received ([9cea2c3](https://github.com/KDAB/qttest-utils/commit/9cea2c3dd4b5798f7f6f0bf382e5eca1694f0eb2))
+* Use tap-parser instead of regexp to parse tap files ([3d9f1f5](https://github.com/KDAB/qttest-utils/commit/3d9f1f5abc77d2af1a57ba6a75b89c8a3ad424ed))

package/CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 ## Prepare your development environment
 
-Just follow `.devcontainer/Dockerfile` to see what's needed.<br?>
+Just follow `.devcontainer/Dockerfile` to see what's needed.<br>
 Basically it's just `nodejs` >= v18 `tsc` and `npm`. <br>
 Qt5, cmake and ninja if you want to run the tests.
 
@@ -10,8 +10,7 @@ Qt5, cmake and ninja if you want to run the tests.
 ## Running tests
 
 ```bash
-tsc
-node out/test.js
+sh test.sh
 ```
 
 Or simply let GitHub actions run the tests for you.<br>

package/Changelog

@@ -2,6 +2,16 @@
 
 
 
+## [2.2.2] - 2024-05-02
+
+### 🐛 Bug Fixes
+
+- Filter out weird tests
+
+### ⚙️ Miscellaneous Tasks
+
+- Code format cmake.ts
+
 ## [2.2.1] - 2024-05-02
 
 ### 🐛 Bug Fixes
@@ -18,6 +28,7 @@
 
 - Ran code format on tests
 - Run codeformat on qttest.ts
+- Bump version
 
 ## [2.2.0] - 2024-04-25
 

package/out/cmake.js

@@ -42,14 +42,23 @@ class CMakeTests {
             }
             return new Promise((resolve, reject) => {
                 (0, qttest_1.logMessage)("Running ctest --show-only=json-v1 with cwd=" + this.buildDirPath);
-                const child = (0, child_process_1.spawn)("ctest", ["--show-only=json-v1"], { "cwd": this.buildDirPath });
+                const child = (0, child_process_1.spawn)("ctest", ["--show-only=json-v1"], {
+                    cwd: this.buildDirPath,
+                });
                 let output = "";
                 child.stdout.on("data", (chunk) => {
                     output += chunk.toString();
                 });
-                child.on("exit", (code) => {
+                child.on("close", (code) => {
                     if (code === 0) {
-                        resolve(this.ctestJsonToList(output));
+                        if (output.length == 0) {
+                            console.error("ctestJsonToList: Empty json output. Command was ctest --show-only=json-v1 , in " +
+                                this.buildDirPath);
+                            reject(new Error("Failed to get ctest JSON output"));
+                        }
+                        else {
+                            resolve(this.ctestJsonToList(output));
+                        }
                     }
                     else {
                         reject(new Error("Failed to run ctest"));
@@ -70,6 +79,20 @@ class CMakeTests {
             test.cwd = testJSON.cwd;
             return test;
         });
+        // filter invalid tests:
+        tests = tests.filter((test) => {
+            // pretty print test
+            if (!test.command || test.command.length == 0)
+                return false;
+            let testExecutablePath = test.executablePath();
+            let baseName = path_1.default.basename(testExecutablePath).toLowerCase();
+            if (baseName.endsWith(".exe"))
+                baseName = baseName.substring(0, baseName.length - 4);
+            // People doing complicated things in add_test()
+            if (baseName == "ctest" || baseName === "cmake")
+                return false;
+            return true;
+        });
         return tests;
     }
     /// Returns the cmake target name for the specified executable
@@ -122,7 +145,7 @@ class CMakeTests {
             // files aren't equal!
             return false;
         }
-        const fs = require('fs');
+        const fs = require("fs");
         if (fs.existsSync(file2)) {
             // It's a real file, not bogus.
             return false;
@@ -178,7 +201,8 @@ class CMakeTests {
                 }
             }
         }
-        (0, qttest_1.logMessage)("cppFilesForExecutable: Could not find cpp files for executable " + executable);
+        (0, qttest_1.logMessage)("cppFilesForExecutable: Could not find cpp files for executable " +
+            executable);
         return [];
     }
 }

package/out/qttest.js

@@ -43,6 +43,7 @@ const child_process_1 = require("child_process");
 const path_1 = __importDefault(require("path"));
 const fs = __importStar(require("fs"));
 const cmake_1 = require("./cmake");
+const tap_parser_1 = require("tap-parser");
 var gLogFunction;
 function logMessage(message) {
     if (gLogFunction) {
@@ -211,8 +212,8 @@ class QtTest {
         return undefined;
     }
     /// Runs this test
-    runTest(slot, cwd = "") {
-        return __awaiter(this, void 0, void 0, function* () {
+    runTest(slot_1) {
+        return __awaiter(this, arguments, void 0, function* (slot, cwd = "") {
             let args = [];
             if (slot) {
                 // Runs a single Qt test instead
@@ -255,7 +256,12 @@ class QtTest {
                     if (this.slots && this.slots.length > 0) {
                         /// When running a QtTest executable, let's check which sub-tests failed
                         /// (So VSCode can show some error icon for each fail)
-                        yield this.updateSubTestStates(cwdDir, slot);
+                        try {
+                            yield this.updateSubTestStates(cwdDir, slot);
+                        }
+                        catch (e) {
+                            logMessage("Failed to update sub-test states: " + e);
+                        }
                     }
                     if (code === 0) {
                         resolve(true);
@@ -297,15 +303,48 @@ class QtTest {
                         reject(error);
                     }
                     else {
-                        // A fail line is something like:
-                        // at: MyTest::testF() (/some/path/qttest-utils/test/qt_test/test2.cpp:13)
-                        const pattern = /at:\s+(.+?)::(.+?)\(\)\s+\((.+?):(\d+)\)/gm;
-                        const matches = Array.from(data.matchAll(pattern));
-                        const failedResults = matches.map((match) => ({
-                            name: match[2],
-                            filePath: match[3],
-                            lineNumber: parseInt(match[4]),
-                        }));
+                        let failedResults = [];
+                        try {
+                            const tap_events = tap_parser_1.Parser.parse(data);
+                            for (let event of tap_events) {
+                                try {
+                                    if (event.length < 2)
+                                        continue;
+                                    if (event.at(0) != "assert")
+                                        continue;
+                                    var obj = event.at(1);
+                                    let pass = obj["ok"] === true;
+                                    let xfail = !pass && obj["todo"] !== false;
+                                    if (xfail) {
+                                        // This is a QEXPECT_FAIL test, all good.
+                                        // QtTest outputs it as "todo"
+                                        continue;
+                                    }
+                                    // There's an QEXPECT_FAIL but test passed, not good.
+                                    let xpass = pass && obj["todo"].includes("returned TRUE unexpectedly");
+                                    if (!pass || xpass) {
+                                        // We found a failure
+                                        var name = obj["name"].replace(/\(.*\)/, "");
+                                        var filename = "";
+                                        var lineNumber = -1;
+                                        if (obj["diag"]) {
+                                            filename = obj["diag"]["file"];
+                                            lineNumber = obj["diag"]["line"];
+                                        }
+                                        else {
+                                            // A XPASS for example misses file:line info. Nothing we can do, it's a Qt bug arguably.
+                                        }
+                                        failedResults.push({
+                                            name: name,
+                                            filePath: filename,
+                                            lineNumber: lineNumber,
+                                        });
+                                    }
+                                }
+                                catch (e) { }
+                            }
+                        }
+                        catch (e) { }
                         resolve(failedResults);
                     }
                 });

package/out/test.js

@@ -57,8 +57,14 @@ function runTests(buildDirPath) {
         // 2. Test that the discovered slots are correct:
         yield qt.dumpTestSlots();
         let expected_slots = {
-            "test/qt_test/build-dev/test1": ["testA", "testB", "testC"],
-            "test/qt_test/build-dev/test2": ["testD", "testE", "testF"],
+            "test/qt_test/build-dev/test1": ["testA", "testB", "testC", "testXFAIL"],
+            "test/qt_test/build-dev/test2": [
+                "testD",
+                "testE",
+                "testF",
+                "testXPASS",
+                "testMixXFAILWithFAIL",
+            ],
             "test/qt_test/build-dev/test3": ["testAbortsEverythig", "testH", "testI"],
         };
         for (var executable of qt.qtTestExecutables) {
@@ -122,6 +128,39 @@ function runTests(buildDirPath) {
             console.error("Expected 0 executables, got " + executables.length);
             process.exit(1);
         }
+        // 6. Run a slot that has XFAIL
+        slot = qt.qtTestExecutables[0].slots[3];
+        if (slot.name != "testXFAIL") {
+            console.error("Expected slot name to be testXFAIL");
+            process.exit(1);
+        }
+        yield slot.runTest();
+        if (slot.lastTestFailure) {
+            console.error("Expected test to pass: " + slot.name);
+            process.exit(1);
+        }
+        // 7. Run a slot that has XPASS
+        slot = qt.qtTestExecutables[1].slots[3];
+        if (slot.name != "testXPASS") {
+            console.error("Expected slot name to be testXPASS");
+            process.exit(1);
+        }
+        yield slot.runTest();
+        if (!slot.lastTestFailure) {
+            console.error("Expected test to fail: " + slot.name);
+            process.exit(1);
+        }
+        // 8. Run a slot that has both XFAIL and FAIL
+        slot = qt.qtTestExecutables[1].slots[4];
+        if (slot.name != "testMixXFAILWithFAIL") {
+            console.error("Expected slot name to be testMixXFAILWithFAIL");
+            process.exit(1);
+        }
+        yield slot.runTest();
+        if (!slot.lastTestFailure) {
+            console.error("Expected test to fail: " + slot.name);
+            process.exit(1);
+        }
     });
 }
 function runCodeModelTests(codeModelFile) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iamsergio/qttest-utils",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "description": "API for listing QtTest executables from a build directory and which individual test slots each executable contains. Useful for a Text Explorer VSCode extension.",
   "repository": {
     "type": "git",
@@ -22,5 +22,8 @@
     "@types/node": "^18.15.0",
     "ts-node": "^10.9.1",
     "typescript": "^5.4.4"
+  },
+  "dependencies": {
+    "tap-parser": "^16.0.1"
   }
 }

package/release-please-config.json

@@ -0,0 +1,16 @@
+{
+  "packages": {
+    ".": {
+      "package-name": "",
+      "component": "",
+      "changelog-path": "Changelog",
+      "release-type": "node",
+      "bump-minor-pre-major": false,
+      "bump-patch-for-minor-pre-major": false,
+      "draft": false,
+      "prerelease": false,
+      "pull-request-title-pattern": "chore: Bump to v${version}"
+    }
+  },
+  "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
+}

package/test.sh

@@ -0,0 +1,7 @@
+cmake --preset=dev -S test/qt_test && \
+cmake --build test/qt_test/build-dev && \
+tsc && node out/test.js && \
+rm -rf test/qt_test/build-dev && \
+cmake --preset=dev6 -S test/qt_test && \
+cmake --build test/qt_test/build-dev && \
+tsc && node out/test.js