@iamsergio/qttest-utils 0.4.5 → 0.4.6

package/README.md

@@ -4,4 +4,14 @@ A nodejs module for listing Qt Test executables and their individual test slots
 
 To be used by vscode extensions that implement the `Testing API`, but can also be used standalone for whatever reason ;).
 
-For example usage `src/example.ts`.
+
+## Example
+
+```
+$ cd test/qt_test
+$ cmake --preset=dev
+$ cmake --build build-dev/
+$ cd ../..
+$ tsc
+$ node out/example.js test/qt_test/build-dev
+```

package/a

@@ -0,0 +1,8 @@
+const pattern = /at:\s+(.+?)::(.+?)\s+\((.+?):(\d+)\)/gm;
+        const matches = Array.from(data.matchAll(pattern));
+        const failingTests = matches.map(match => ({
+          name: match[2],
+          filePath: match[3],
+          lineNumber: parseInt(match[4]),
+        }));
+        resolve(failingTests);

package/foo

@@ -0,0 +1,15 @@
+TAP version 13
+# MyTest
+ok 1 - initTestCase()
+not ok 2 - testF()
+  ---
+  # failed
+  at: MyTest::testF() (/data/sources/vscode-test-extension/qttest-utils/test/qt_test/test2.cpp:13)
+  file: /data/sources/vscode-test-extension/qttest-utils/test/qt_test/test2.cpp
+  line: 13
+  ...
+ok 3 - cleanupTestCase()
+1..3
+# tests 3
+# pass 2
+# fail 1

package/out/example.js

@@ -36,10 +36,37 @@ function example() {
         yield qt.removeNonLinking();
         // Example of filtering out by regexp:
         qt.removeMatching(/(tst_view|tst_window)/);
-        // Example of filtering out by regexp (inverted):
-        qt.maintainMatching(/(tst_docks|tst_qtwidgets|tst_multisplitter)/);
+        // Uncomment to see example of filtering out by regexp (inverted):
+        // qt.maintainMatching(/(tst_docks|tst_qtwidgets|tst_multisplitter)/);
         qt.dumpExecutablePaths();
-        qt.dumpTestSlots();
+        yield qt.dumpTestSlots();
+        console.log("\nRunning tests...");
+        for (var executable of qt.qtTestExecutables) {
+            yield executable.runTest();
+            if (executable.lastExitCode === 0)
+                console.log("    PASS: " + executable.filename);
+            else
+                console.log("    FAIL: " + executable.filename + "; code=" + executable.lastExitCode);
+            for (let slot of executable.slots) {
+                if (slot.lastTestFailure) {
+                    console.log("        failed slot=" + slot.name + "; path=" + slot.lastTestFailure.filePath + "; line=" + slot.lastTestFailure.lineNumber);
+                }
+            }
+        }
+        // Also run individual slots, just for example purposes:
+        console.log("\nRunning single tests...");
+        let slot = qt.qtTestExecutables[1].slots[0];
+        yield slot.runTest();
+        if (slot.lastTestFailure)
+            console.log("    FAIL:" + JSON.stringify(slot.lastTestFailure));
+        else
+            console.log("    PASS:");
+        let slot2 = qt.qtTestExecutables[1].slots[2];
+        yield slot2.runTest();
+        if (slot2.lastTestFailure)
+            console.log("    FAIL:" + JSON.stringify(slot2.lastTestFailure));
+        else
+            console.log("    PASS");
     });
 }
 example();

package/out/qttest.d.ts

@@ -7,6 +7,7 @@ export declare class QtTest {
     readonly buildDirPath: string;
     vscodeTestItem: any | undefined;
     slots: QtTestSlot[] | null;
+    lastExitCode: number;
     constructor(filename: string, buildDirPath: string);
     get id(): string;
     get label(): string;
@@ -23,12 +24,17 @@ export declare class QtTest {
      */
     linksToQtTestLib(): Promise<boolean> | undefined;
     isQtTestViaHelp(): Promise<boolean | undefined>;
-    runTest(slotName?: string, cwd?: string): Promise<boolean>;
+    slotByName(name: string): QtTestSlot | undefined;
+    runTest(slot?: QtTestSlot, cwd?: string): Promise<boolean>;
+    tapOutputFileName(slot?: QtTestSlot): string;
+    txtOutputFileName(slot?: QtTestSlot): string;
     command(): {
         label: string;
         executablePath: string;
         args: string[];
     };
+    clearSubTestStates(): void;
+    updateSubTestStates(cwdDir: string, slot?: QtTestSlot): Promise<void>;
 }
 /**
  * Represents a single Qt test slot
@@ -37,6 +43,7 @@ export declare class QtTestSlot {
     name: string;
     parentQTest: QtTest;
     vscodeTestItem: any | undefined;
+    lastTestFailure: TestFailure | undefined;
     constructor(name: string, parent: QtTest);
     get id(): string;
     get absoluteFilePath(): string;
@@ -60,3 +67,8 @@ export declare class QtTests {
     dumpExecutablePaths(): void;
     dumpTestSlots(): Promise<void>;
 }
+export interface TestFailure {
+    name: string;
+    filePath: string;
+    lineNumber: number;
+}

package/out/qttest.js

@@ -49,7 +49,10 @@ const cmake_1 = require("./cmake");
  */
 class QtTest {
     constructor(filename, buildDirPath) {
+        /// The list of individual runnable test slots
         this.slots = null;
+        /// Set after running
+        this.lastExitCode = 0;
         this.filename = filename;
         this.buildDirPath = buildDirPath;
     }
@@ -159,34 +162,110 @@ class QtTest {
             });
         });
     }
+    slotByName(name) {
+        if (!this.slots)
+            return undefined;
+        for (let slot of this.slots) {
+            if (slot.name == name)
+                return slot;
+        }
+        return undefined;
+    }
     /// Runs this test
-    runTest(slotName, cwd = "") {
+    runTest(slot, cwd = "") {
         return __awaiter(this, void 0, void 0, function* () {
             let args = [];
-            if (slotName) {
+            if (slot) {
                 // Runs a single Qt test instead
-                args = args.concat(slotName);
+                args = args.concat(slot.name);
             }
+            else {
+                this.clearSubTestStates();
+            }
+            // log to file
+            args = args.concat("-o").concat(this.tapOutputFileName(slot) + ",tap");
+            args = args.concat("-o").concat(this.txtOutputFileName(slot) + ",txt");
             return yield new Promise((resolve, reject) => {
-                let opts = cwd.length > 0 ? { cwd: cwd } : { cwd: this.buildDirPath };
-                const child = (0, child_process_1.spawn)(this.filename, args, opts);
-                child.stdout.on("data", (chunk) => {
-                    // chunk.toString()
-                });
-                child.on("exit", (code) => {
+                let cwdDir = cwd.length > 0 ? cwd : this.buildDirPath;
+                const child = (0, child_process_1.spawn)(this.filename, args, { cwd: cwdDir });
+                child.on("exit", (code) => __awaiter(this, void 0, void 0, function* () {
+                    /// We can code even be null ?
+                    if (code == undefined)
+                        code = -1;
+                    if (!slot) {
+                        this.lastExitCode = code;
+                    }
+                    /// When running a QtTest executable, let's check which sub-tests failed
+                    /// (So VSCode can show some error icon for each fail)
+                    yield this.updateSubTestStates(cwdDir, slot);
                     if (code === 0) {
                         resolve(true);
                     }
                     else {
                         resolve(false);
                     }
-                });
+                }));
             });
         });
     }
+    /// Using .tap so we don't have to use a separate XML library
+    /// .tap is plain text and a single regexp can catch the failing tests and line number
+    tapOutputFileName(slot) {
+        let slotName = slot ? ("_" + slot.name) : "";
+        return this.label + slotName + ".tap";
+    }
+    txtOutputFileName(slot) {
+        let slotName = slot ? ("_" + slot.name) : "";
+        return this.label + slotName + ".txt";
+    }
     command() {
         return { label: this.label, executablePath: this.filename, args: [] };
     }
+    clearSubTestStates() {
+        if (this.slots) {
+            for (let slot of this.slots) {
+                slot.lastTestFailure = undefined;
+            }
+        }
+    }
+    updateSubTestStates(cwdDir, slot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let tapFileName = cwdDir + "/" + this.tapOutputFileName(slot);
+            var failures = yield new Promise((resolve, reject) => {
+                fs.readFile(tapFileName, "utf8", (error, data) => {
+                    if (error) {
+                        console.log("Failed to read log file");
+                        reject(error);
+                    }
+                    else {
+                        // A fail line is something like:
+                        // at: MyTest::testF() (/some/path/qttest-utils/test/qt_test/test2.cpp:13)
+                        const pattern = /at:\s+(.+?)::(.+?)\(\)\s+\((.+?):(\d+)\)/gm;
+                        const matches = Array.from(data.matchAll(pattern));
+                        const failedResults = matches.map(match => ({
+                            name: match[2],
+                            filePath: match[3],
+                            lineNumber: parseInt(match[4]),
+                        }));
+                        resolve(failedResults);
+                    }
+                });
+            });
+            for (let failure of failures) {
+                if (slot && slot.name != failure.name) {
+                    // We executed a single slot, ignore anything else
+                    continue;
+                }
+                let failedSlot = this.slotByName(failure.name);
+                if (failedSlot) {
+                    failedSlot.lastTestFailure = failure;
+                }
+                else {
+                    console.log("Failed to find slot with name " + failure.name);
+                }
+            }
+        });
+    }
 }
 exports.QtTest = QtTest;
 /**
@@ -205,7 +284,7 @@ class QtTestSlot {
     }
     runTest() {
         return __awaiter(this, void 0, void 0, function* () {
-            return this.parentQTest.runTest(this.name);
+            return this.parentQTest.runTest(this);
         });
     }
     command() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iamsergio/qttest-utils",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "API for listing QtTest executables from a build directory and which individual test slots each executable contains. Useful for a Text Explorer VSCode extension.",
   "repository": {
     "type": "git",

package/src/example.ts

@@ -5,6 +5,7 @@
 import { CMakeTests, CMakeTest } from "./cmake";
 import { QtTest, QtTests } from "./qttest";
 import fs from 'fs';
+import { exec } from "child_process";
 
 async function example() {
     const args = process.argv.slice(2)
@@ -31,11 +32,43 @@ async function example() {
     // Example of filtering out by regexp:
     qt.removeMatching(/(tst_view|tst_window)/);
 
-    // Example of filtering out by regexp (inverted):
-    qt.maintainMatching(/(tst_docks|tst_qtwidgets|tst_multisplitter)/);
+    // Uncomment to see example of filtering out by regexp (inverted):
+    // qt.maintainMatching(/(tst_docks|tst_qtwidgets|tst_multisplitter)/);
 
     qt.dumpExecutablePaths();
-    qt.dumpTestSlots();
+    await qt.dumpTestSlots();
+
+    console.log("\nRunning tests...");
+    for (var executable of qt.qtTestExecutables) {
+        await executable.runTest();
+        if (executable.lastExitCode === 0)
+            console.log("    PASS: " + executable.filename);
+        else
+            console.log("    FAIL: " + executable.filename + "; code=" + executable.lastExitCode);
+        for (let slot of executable.slots!) {
+            if (slot.lastTestFailure) {
+                console.log("        failed slot=" + slot.name + "; path=" + slot.lastTestFailure.filePath + "; line=" + slot.lastTestFailure.lineNumber);
+            }
+        }
+    }
+
+    // Also run individual slots, just for example purposes:
+
+    console.log("\nRunning single tests...");
+    let slot = qt.qtTestExecutables[1].slots![0];
+    await slot.runTest();
+    if (slot.lastTestFailure)
+        console.log("    FAIL:" + JSON.stringify(slot.lastTestFailure));
+    else
+        console.log("    PASS:");
+
+    let slot2 = qt.qtTestExecutables[1].slots![2];
+    await slot2.runTest();
+    if (slot2.lastTestFailure)
+        console.log("    FAIL:" + JSON.stringify(slot2.lastTestFailure));
+    else
+        console.log("    PASS");
+
 }
 
 example();

package/src/qttest.ts

@@ -18,8 +18,12 @@ export class QtTest {
     /// Allows vscode extensions to associate with a test item
     vscodeTestItem: any | undefined;
 
+    /// The list of individual runnable test slots
     slots: QtTestSlot[] | null = null;
 
+    /// Set after running
+    lastExitCode: number = 0;
+
     constructor(filename: string, buildDirPath: string) {
         this.filename = filename;
         this.buildDirPath = buildDirPath;
@@ -138,22 +142,49 @@ export class QtTest {
         });
     }
 
+    public slotByName(name: string): QtTestSlot | undefined {
+        if (!this.slots)
+            return undefined;
+
+        for (let slot of this.slots) {
+            if (slot.name == name)
+                return slot;
+        }
+
+        return undefined;
+    }
+
     /// Runs this test
-    public async runTest(slotName?: string, cwd: string = ""): Promise<boolean> {
+    public async runTest(slot?: QtTestSlot, cwd: string = ""): Promise<boolean> {
         let args: string[] = [];
-        if (slotName) {
+        if (slot) {
             // Runs a single Qt test instead
-            args = args.concat(slotName);
+            args = args.concat(slot.name);
+        } else {
+            this.clearSubTestStates();
         }
 
+        // log to file
+        args = args.concat("-o").concat(this.tapOutputFileName(slot) + ",tap");
+        args = args.concat("-o").concat(this.txtOutputFileName(slot) + ",txt");
+
         return await new Promise((resolve, reject) => {
-            let opts = cwd.length > 0 ? { cwd: cwd } : { cwd: this.buildDirPath };
-            const child = spawn(this.filename, args, opts);
-            child.stdout.on("data", (chunk) => {
-                // chunk.toString()
-            });
+            let cwdDir = cwd.length > 0 ? cwd : this.buildDirPath;
+            const child = spawn(this.filename, args, { cwd: cwdDir });
+
+            child.on("exit", async (code) => {
+
+                /// We can code even be null ?
+                if (code == undefined) code = -1;
+
+                if (!slot) {
+                    this.lastExitCode = code;
+                }
+
+                /// When running a QtTest executable, let's check which sub-tests failed
+                /// (So VSCode can show some error icon for each fail)
+                await this.updateSubTestStates(cwdDir, slot);
 
-            child.on("exit", (code) => {
                 if (code === 0) {
                     resolve(true);
                 } else {
@@ -163,9 +194,71 @@ export class QtTest {
         });
     }
 
+    /// Using .tap so we don't have to use a separate XML library
+    /// .tap is plain text and a single regexp can catch the failing tests and line number
+    public tapOutputFileName(slot?: QtTestSlot): string {
+        let slotName = slot ? ("_" + slot.name) : "";
+        return this.label + slotName + ".tap";
+    }
+
+    public txtOutputFileName(slot?: QtTestSlot): string {
+        let slotName = slot ? ("_" + slot.name) : "";
+        return this.label + slotName + ".txt";
+    }
+
     public command(): { label: string, executablePath: string, args: string[] } {
         return { label: this.label, executablePath: this.filename, args: [] };
     }
+
+
+    public clearSubTestStates() {
+        if (this.slots) {
+            for (let slot of this.slots) {
+                slot.lastTestFailure = undefined;
+            }
+        }
+    }
+
+    public async updateSubTestStates(cwdDir: string, slot?: QtTestSlot) {
+
+        let tapFileName: string = cwdDir + "/" + this.tapOutputFileName(slot);
+
+        var failures = await new Promise<TestFailure[]>((resolve, reject) => {
+            fs.readFile(tapFileName, "utf8", (error, data) => {
+                if (error) {
+                    console.log("Failed to read log file");
+                    reject(error);
+                } else {
+                    // A fail line is something like:
+                    // at: MyTest::testF() (/some/path/qttest-utils/test/qt_test/test2.cpp:13)
+
+                    const pattern = /at:\s+(.+?)::(.+?)\(\)\s+\((.+?):(\d+)\)/gm;
+                    const matches = Array.from(data.matchAll(pattern));
+                    const failedResults = matches.map(match => ({
+                        name: match[2],
+                        filePath: match[3],
+                        lineNumber: parseInt(match[4]),
+                    }));
+
+                    resolve(failedResults);
+                }
+            });
+        });
+
+        for (let failure of failures) {
+            if (slot && slot.name != failure.name) {
+                // We executed a single slot, ignore anything else
+                continue;
+            }
+
+            let failedSlot = this.slotByName(failure.name);
+            if (failedSlot) {
+                failedSlot.lastTestFailure = failure;
+            } else {
+                console.log("Failed to find slot with name " + failure.name);
+            }
+        }
+    }
 }
 
 /**
@@ -180,6 +273,9 @@ export class QtTestSlot {
     /// Allows vscode extensions to associate with a test item
     vscodeTestItem: any | undefined;
 
+    /// Set after running
+    lastTestFailure: TestFailure | undefined;
+
     constructor(name: string, parent: QtTest) {
         this.name = name;
         this.parentQTest = parent;
@@ -194,7 +290,7 @@ export class QtTestSlot {
     }
 
     public async runTest(): Promise<boolean> {
-        return this.parentQTest.runTest(this.name);
+        return this.parentQTest.runTest(this);
     }
 
     public command(): { label: string, executablePath: string, args: string[] } {
@@ -273,3 +369,10 @@ export class QtTests {
         }
     }
 }
+
+/// Represents a failure location
+export interface TestFailure {
+    name: string;
+    filePath: string;
+    lineNumber: number;
+}

package/test/qt_test/CMakeLists.txt

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: 2023 Klarälvdalens Datakonsult AB, a KDAB Group company <info@kdab.com>
+# Author: Sergio Martins <sergio.martins@kdab.com>
+# SPDX-License-Identifier: MIT
+
+cmake_minimum_required(VERSION 3.12)
+project(qttest)
+
+find_package(Qt5Test 5.15 REQUIRED)
+set(CMAKE_AUTOMOC ON)
+
+add_executable(test1 test1.cpp)
+add_executable(test2 test2.cpp)
+add_executable(test3 test3.cpp)
+
+target_link_libraries(test1 Qt5::Test)
+target_link_libraries(test2 Qt5::Test)
+target_link_libraries(test3 Qt5::Test)
+
+enable_testing()
+add_test(NAME test1 COMMAND test1)
+add_test(NAME test2 COMMAND test2)
+add_test(NAME test3 COMMAND test3)

package/test/qt_test/CMakePresets.json

@@ -0,0 +1,15 @@
+{
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "dev",
+            "displayName": "dev",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build-dev",
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Debug",
+                "CMAKE_EXPORT_COMPILE_COMMANDS": "ON"
+            }
+        }
+    ]
+}

package/test/qt_test/test1.cpp

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Klarälvdalens Datakonsult AB, a KDAB Group
+// company <info@kdab.com> Author: Sergio Martins <sergio.martins@kdab.com>
+// SPDX-License-Identifier: MIT
+
+#include <QObject>
+#include <QtTest>
+
+class MyTest : public QObject {
+  Q_OBJECT
+private Q_SLOTS:
+  void testA() {}
+  void testB() {}
+  void testC() {}
+};
+
+QTEST_MAIN(MyTest);
+
+#include <test1.moc>

package/test/qt_test/test2.cpp

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Klarälvdalens Datakonsult AB, a KDAB Group
+// company <info@kdab.com> Author: Sergio Martins <sergio.martins@kdab.com>
+// SPDX-License-Identifier: MIT
+
+#include <QObject>
+#include <QtTest>
+
+class MyTest : public QObject {
+  Q_OBJECT
+private Q_SLOTS:
+  void testD() {}
+  void testE() {}
+  void testF() { QFAIL("failed"); }
+};
+
+QTEST_MAIN(MyTest);
+
+#include <test2.moc>

package/test/qt_test/test3.cpp

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2023 Klarälvdalens Datakonsult AB, a KDAB Group
+// company <info@kdab.com> Author: Sergio Martins <sergio.martins@kdab.com>
+// SPDX-License-Identifier: MIT
+
+#include <QObject>
+#include <QtTest>
+
+class MyTest : public QObject {
+  Q_OBJECT
+private Q_SLOTS:
+  void testG() {}
+  void testH() {}
+  void testI() {}
+};
+
+QTEST_MAIN(MyTest);
+
+#include <test3.moc>

package/test1.log

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TestCase name="MyTest">
+<Environment>
+    <QtVersion>5.15.2</QtVersion>
+    <QtBuild>Qt 5.15.2 (x86_64&#x002D;little_endian&#x002D;lp64 shared (dynamic) release build; by GCC 5.3.1 20160406 (Red Hat 5.3.1&#x002D;6))</QtBuild>
+    <QTestVersion>5.15.2</QTestVersion>
+</Environment>
+<TestFunction name="initTestCase">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.312962"/>
+</TestFunction>
+<TestFunction name="testD">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.010127"/>
+</TestFunction>
+<TestFunction name="testE">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.005239"/>
+</TestFunction>
+<TestFunction name="testF">
+<Incident type="fail" file="/data/sources/vscode&#x002D;test&#x002D;extension/qttest&#x002D;utils/test/qt_test/test2.cpp" line="13">
+    <Description><![CDATA[failed]]></Description>
+</Incident>
+    <Duration msecs="0.018438"/>
+</TestFunction>
+<TestFunction name="cleanupTestCase">
+<Incident type="pass" file="" line="0" />
+    <Duration msecs="0.00426"/>
+</TestFunction>
+<Duration msecs="0.432951"/>
+</TestCase>

package/test1.log1

@@ -0,0 +1,17 @@
+TAP version 13
+# MyTest
+ok 1 - initTestCase()
+ok 2 - testD()
+ok 3 - testE()
+not ok 4 - testF()
+  ---
+  # failed
+  at: MyTest::testF() (/data/sources/vscode-test-extension/qttest-utils/test/qt_test/test2.cpp:13)
+  file: /data/sources/vscode-test-extension/qttest-utils/test/qt_test/test2.cpp
+  line: 13
+  ...
+ok 5 - cleanupTestCase()
+1..5
+# tests 5
+# pass 4
+# fail 1