@iobroker/testing 2.5.6 → 3.0.1

package/CHANGELOG.md

@@ -4,6 +4,13 @@
 	PLACEHOLDER for next version:
 	## __WORK IN PROGRESS__
 -->
+## 3.0.1 (2022-05-09)
+* BREAKING: The function signature of `defineAdditionalTests` in integration tests has changed. All user-defined integration tests must now be grouped in one or more `suite` blocks. The adapter will now only be started at the beginning of each suite. See the documentation for details.
+* BREAKING: The function signature of `harness.startAdapterAndWait` has changed. It now accepts a boolean as the first parameter which controls whether to wait for the `alive` state (`false`) or the `info.connection` state (`true`).
+
+## 2.6.0 (2022-04-18)
+* The loglevel for the adapter and DB instances is now configurable and defaults to `"debug"` in both cases
+
 ## 2.5.6 (2022-03-05)
 * Allow immediate exit with code `0` for `once` and `subscribe` adapters too
 

package/README.md

@@ -26,29 +26,6 @@ tests.packageFiles(path.join(__dirname, ".."));
 // This should be the adapter's root directory
 ```
 
-### Adapter startup (Unit test)
-
-**Unit tests for adapter startup were removed and are essentially a no-op now.**  
-If you defined your own tests, they should still work.
-
-```ts
-const path = require("path");
-const { tests } = require("@iobroker/testing");
-
-tests.unit(path.join(__dirname, ".."), {
-	//     ~~~~~~~~~~~~~~~~~~~~~~~~~
-	// This should be the adapter's root directory
-
-	// Define your own tests inside defineAdditionalTests.
-	// If you need predefined objects etc. here, you need to take care of it yourself
-	defineAdditionalTests() {
-		it("works", () => {
-			// see below how these could look like
-		});
-	},
-});
-```
-
 ### Adapter startup (Integration test)
 
 Run the following snippet in a `mocha` test file to test the adapter startup process against a real JS-Controller instance:
@@ -67,13 +44,15 @@ tests.integration(path.join(__dirname, ".."), {
 	allowedExitCodes: [11],
 
 	// Define your own tests inside defineAdditionalTests
-	// Since the tests are heavily instrumented, you need to create and use a so called "harness" to control the tests.
-	defineAdditionalTests(getHarness) {
-		describe("Test sendTo()", () => {
+	defineAdditionalTests({ suite }) {
+		// All tests (it, describe) must be grouped in one or more suites. Each suite sets up a fresh environment for the adapter tests.
+		// At the beginning of each suite, the databases will be reset and the adapter will be started.
+		// The adapter will run until the end of each suite.
+
+		// Since the tests are heavily instrumented, each suite gives access to a so called "harness" to control the tests.
+		suite("Test sendTo()", (harness) => {
 			it("Should work", () => {
 				return new Promise(async (resolve) => {
-					// Create a fresh harness instance each test!
-					const harness = getHarness();
 					// Start the adapter and wait until it has started
 					await harness.startAdapterAndWait();
 
@@ -89,6 +68,29 @@ tests.integration(path.join(__dirname, ".."), {
 });
 ```
 
+### Adapter startup (Unit test)
+
+**Unit tests for adapter startup were removed and are essentially a no-op now.**  
+If you defined your own tests, they should still work.
+
+```ts
+const path = require("path");
+const { tests } = require("@iobroker/testing");
+
+tests.unit(path.join(__dirname, ".."), {
+	//     ~~~~~~~~~~~~~~~~~~~~~~~~~
+	// This should be the adapter's root directory
+
+	// Define your own tests inside defineAdditionalTests.
+	// If you need predefined objects etc. here, you need to take care of it yourself
+	defineAdditionalTests() {
+		it("works", () => {
+			// see below how these could look like
+		});
+	},
+});
+```
+
 ### Helper functions for your own tests
 
 Under `utils`, several functions are exposed to use in your own tests:

package/build/tests/integration/index.d.ts

@@ -1,9 +1,24 @@
+/// <reference types="iobroker" />
+/// <reference types="mocha" />
 import { TestHarness } from "./lib/harness";
 export interface TestAdapterOptions {
     allowedExitCodes?: (number | string)[];
+    /** The loglevel to use for DB and adapter related logs */
+    loglevel?: ioBroker.LogLevel;
     /** How long to wait before the adapter startup is considered successful */
     waitBeforeStartupSuccess?: number;
     /** Allows you to define additional tests */
-    defineAdditionalTests?: (getHarness: () => TestHarness) => void;
+    defineAdditionalTests?: (args: TestContext) => void;
+}
+export interface TestContext {
+    /**
+     * Defines a test suite. At the start of each suite, the adapter will be started with a fresh environment.
+     * To define tests in each suite, use describe and it as usual.
+     *
+     * Each suite has its own test harness, which gets passed as an argument.
+     */
+    suite: (name: string, fn: (harness: TestHarness) => void) => void;
+    describe: Mocha.SuiteFunction;
+    it: Mocha.TestFunction;
 }
 export declare function testAdapter(adapterDir: string, options?: TestAdapterOptions): void;

package/build/tests/integration/index.js

@@ -32,6 +32,7 @@ const adapterSetup_1 = require("./lib/adapterSetup");
 const controllerSetup_1 = require("./lib/controllerSetup");
 const dbConnection_1 = require("./lib/dbConnection");
 const harness_1 = require("./lib/harness");
+const logger_1 = require("./lib/logger");
 function testAdapter(adapterDir, options = {}) {
     const appName = (0, adapterTools_1.getAppName)(adapterDir);
     const adapterName = (0, adapterTools_1.getAdapterName)(adapterDir);
@@ -40,111 +41,162 @@ function testAdapter(adapterDir, options = {}) {
     let dbConnection;
     let harness;
     const controllerSetup = new controllerSetup_1.ControllerSetup(adapterDir, testDir);
+    let objectsBackup;
+    let statesBackup;
+    let isInSuite = false;
     console.log();
     console.log(`Running tests in ${testDir}`);
     console.log();
-    describe(`Test the adapter (in a live environment)`, () => {
-        let objectsBackup;
-        let statesBackup;
-        before(async function () {
-            // Installation may take a while - especially if rsa-compat needs to be installed
-            const oneMinute = 60000;
-            this.timeout(30 * oneMinute);
-            if (await controllerSetup.isJsControllerRunning()) {
-                throw new Error("JS-Controller is already running! Stop it for the first test run and try again!");
-            }
-            const adapterSetup = new adapterSetup_1.AdapterSetup(adapterDir, testDir);
-            // Installation happens in two steps:
-            // First we need to set up JS Controller, so the databases etc. can be created
-            // First we need to copy all files and execute an npm install
-            await controllerSetup.prepareTestDir();
-            // Only then we can install the adapter, because some (including VIS) try to access
-            // the databases if JS Controller is installed
-            await adapterSetup.installAdapterInTestDir();
-            const dbConnection = new dbConnection_1.DBConnection(appName, testDir);
-            await dbConnection.start();
-            controllerSetup.setupSystemConfig(dbConnection);
-            await controllerSetup.disableAdminInstances(dbConnection);
-            await adapterSetup.deleteOldInstances(dbConnection);
-            await adapterSetup.addAdapterInstance();
-            await dbConnection.stop();
-            // Create a copy of the databases that we can restore later
-            ({ objects: objectsBackup, states: statesBackup } =
-                await dbConnection.backup());
+    async function prepareTests() {
+        var _a;
+        // Installation may take a while - especially if rsa-compat needs to be installed
+        const oneMinute = 60000;
+        this.timeout(30 * oneMinute);
+        if (await controllerSetup.isJsControllerRunning()) {
+            throw new Error("JS-Controller is already running! Stop it for the first test run and try again!");
+        }
+        const adapterSetup = new adapterSetup_1.AdapterSetup(adapterDir, testDir);
+        // Installation happens in two steps:
+        // First we need to set up JS Controller, so the databases etc. can be created
+        // First we need to copy all files and execute an npm install
+        await controllerSetup.prepareTestDir();
+        // Only then we can install the adapter, because some (including VIS) try to access
+        // the databases if JS Controller is installed
+        await adapterSetup.installAdapterInTestDir();
+        const dbConnection = new dbConnection_1.DBConnection(appName, testDir, (0, logger_1.createLogger)((_a = options.loglevel) !== null && _a !== void 0 ? _a : "debug"));
+        await dbConnection.start();
+        controllerSetup.setupSystemConfig(dbConnection);
+        await controllerSetup.disableAdminInstances(dbConnection);
+        await adapterSetup.deleteOldInstances(dbConnection);
+        await adapterSetup.addAdapterInstance();
+        await dbConnection.stop();
+        // Create a copy of the databases that we can restore later
+        ({ objects: objectsBackup, states: statesBackup } =
+            await dbConnection.backup());
+    }
+    async function shutdownTests() {
+        // Stopping the processes may take a while
+        this.timeout(30000);
+        // Stop the controller again
+        await harness.stopController();
+        harness.removeAllListeners();
+    }
+    async function resetDbAndStartHarness() {
+        var _a, _b;
+        this.timeout(30000);
+        dbConnection = new dbConnection_1.DBConnection(appName, testDir, (0, logger_1.createLogger)((_a = options.loglevel) !== null && _a !== void 0 ? _a : "debug"));
+        // Clean up before every single test
+        await Promise.all([
+            controllerSetup.clearDBDir(),
+            controllerSetup.clearLogDir(),
+            dbConnection.restore(objectsBackup, statesBackup),
+        ]);
+        // Create a new test harness
+        await dbConnection.start();
+        harness = new harness_1.TestHarness(adapterDir, testDir, dbConnection);
+        // Enable the adapter and set its loglevel to the selected one
+        await harness.changeAdapterConfig(adapterName, {
+            common: {
+                enabled: true,
+                loglevel: (_b = options.loglevel) !== null && _b !== void 0 ? _b : "debug",
+            },
         });
-        beforeEach(async function () {
-            this.timeout(30000);
-            dbConnection = new dbConnection_1.DBConnection(appName, testDir);
-            // Clean up before every single test
-            await Promise.all([
-                controllerSetup.clearDBDir(),
-                controllerSetup.clearLogDir(),
-                dbConnection.restore(objectsBackup, statesBackup),
-            ]);
-            // Create a new test harness
-            await dbConnection.start();
-            harness = new harness_1.TestHarness(adapterDir, testDir, dbConnection);
-            // Enable the adapter and set its loglevel to debug
-            await harness.changeAdapterConfig(adapterName, {
-                common: {
-                    enabled: true,
-                    loglevel: "debug",
-                },
+        // And enable the sendTo emulation
+        await harness.enableSendTo();
+    }
+    describe(`Adapter integration tests`, () => {
+        before(prepareTests);
+        describe("Adapter startup", () => {
+            beforeEach(resetDbAndStartHarness);
+            afterEach(shutdownTests);
+            it("The adapter starts", function () {
+                var _a;
+                this.timeout(60000);
+                const allowedExitCodes = new Set((_a = options.allowedExitCodes) !== null && _a !== void 0 ? _a : []);
+                // Adapters with these modes are allowed to "immediately" exit with code 0
+                switch (harness.getAdapterExecutionMode()) {
+                    case "schedule":
+                    case "once":
+                    case "subscribe":
+                        allowedExitCodes.add(0);
+                }
+                return new Promise((resolve, reject) => {
+                    // Register a handler to check the alive state and exit codes
+                    harness
+                        .on("stateChange", async (id, state) => {
+                        if (id ===
+                            `system.adapter.${adapterName}.0.alive` &&
+                            state &&
+                            state.val === true) {
+                            // Wait a bit so we can catch errors that do not happen immediately
+                            await (0, async_1.wait)(options.waitBeforeStartupSuccess !=
+                                undefined
+                                ? options.waitBeforeStartupSuccess
+                                : 5000);
+                            resolve(`The adapter started successfully.`);
+                        }
+                    })
+                        .on("failed", (code) => {
+                        if (!allowedExitCodes.has(code)) {
+                            reject(new Error(`The adapter startup was interrupted unexpectedly with ${typeof code === "number"
+                                ? "code"
+                                : "signal"} ${code}`));
+                        }
+                        else {
+                            // This was a valid exit code
+                            resolve(`The expected ${typeof code === "number"
+                                ? "exit code"
+                                : "signal"} ${code} was received.`);
+                        }
+                    });
+                    harness.startAdapter();
+                }).then((msg) => console.log(msg));
             });
-            // And enable the sendTo emulation
-            await harness.enableSendTo();
-        });
-        afterEach(async function () {
-            // Stopping the processes may take a while
-            this.timeout(30000);
-            // Stop the controller again
-            await harness.stopController();
-            harness.removeAllListeners();
-        });
-        it("The adapter starts", function () {
-            var _a;
-            this.timeout(60000);
-            const allowedExitCodes = new Set((_a = options.allowedExitCodes) !== null && _a !== void 0 ? _a : []);
-            // Adapters with these modes are allowed to "immediately" exit with code 0
-            switch (harness.getAdapterExecutionMode()) {
-                case "schedule":
-                case "once":
-                case "subscribe":
-                    allowedExitCodes.add(0);
-            }
-            return new Promise((resolve, reject) => {
-                // Register a handler to check the alive state and exit codes
-                harness
-                    .on("stateChange", async (id, state) => {
-                    if (id === `system.adapter.${adapterName}.0.alive` &&
-                        state &&
-                        state.val === true) {
-                        // Wait a bit so we can catch errors that do not happen immediately
-                        await (0, async_1.wait)(options.waitBeforeStartupSuccess != undefined
-                            ? options.waitBeforeStartupSuccess
-                            : 5000);
-                        resolve(`The adapter started successfully.`);
-                    }
-                })
-                    .on("failed", (code) => {
-                    if (!allowedExitCodes.has(code)) {
-                        reject(new Error(`The adapter startup was interrupted unexpectedly with ${typeof code === "number"
-                            ? "code"
-                            : "signal"} ${code}`));
-                    }
-                    else {
-                        // This was a valid exit code
-                        resolve(`The expected ${typeof code === "number"
-                            ? "exit code"
-                            : "signal"} ${code} was received.`);
-                    }
-                });
-                harness.startAdapter();
-            }).then((msg) => console.log(msg));
         });
         // Call the user's tests
         if (typeof options.defineAdditionalTests === "function") {
-            options.defineAdditionalTests(() => harness);
+            const originalIt = global.it;
+            // Ensure no it() gets called outside of a suite()
+            function assertSuite() {
+                if (!isInSuite) {
+                    throw new Error("In user-defined adapter tests, it() must NOT be called outside of a suite()");
+                }
+            }
+            const patchedIt = new Proxy(originalIt, {
+                apply(target, thisArg, args) {
+                    assertSuite();
+                    return target.apply(thisArg, args);
+                },
+                get(target, propKey) {
+                    assertSuite();
+                    return target[propKey];
+                },
+            });
+            describe("User-defined tests", () => {
+                // patch the global it() function so nobody can bypass the checks
+                global.it = patchedIt;
+                const lazyHarness = new Proxy({}, {
+                    get(target, propKey) {
+                        return harness[propKey];
+                    },
+                });
+                const args = {
+                    // a test suite is a special describe which sets up and tears down the test environment before and after ALL tests
+                    suite: (name, fn) => {
+                        describe(name, () => {
+                            isInSuite = true;
+                            before(resetDbAndStartHarness);
+                            fn(lazyHarness);
+                            after(shutdownTests);
+                            isInSuite = false;
+                        });
+                    },
+                    describe,
+                    it: patchedIt,
+                };
+                options.defineAdditionalTests(args);
+                global.it = originalIt;
+            });
         }
     });
 }

package/build/tests/integration/lib/dbConnection.d.ts

@@ -11,11 +11,12 @@ export interface DBConnection {
 export declare class DBConnection extends EventEmitter {
     private appName;
     private testDir;
+    private logger;
     /**
      * @param appName The branded name of "iobroker"
      * @param testDir The directory the integration tests are executed in
      */
-    constructor(appName: string, testDir: string);
+    constructor(appName: string, testDir: string, logger: ioBroker.Logger);
     private testDataDir;
     private testControllerDir;
     private _objectsServer;

package/build/tests/integration/lib/dbConnection.js

@@ -33,24 +33,17 @@ const fs_extra_1 = require("fs-extra");
 const path = __importStar(require("path"));
 const tools_1 = require("./tools");
 const debug = (0, debug_1.default)("testing:integration:DBConnection");
-/** The logger instance for the objects and states DB */
-const logger = {
-    silly: console.log,
-    debug: console.log,
-    info: console.log,
-    warn: console.warn,
-    error: console.error,
-};
 /** The DB connection capsules access to the states and objects DB */
 class DBConnection extends events_1.default {
     /**
      * @param appName The branded name of "iobroker"
      * @param testDir The directory the integration tests are executed in
      */
-    constructor(appName, testDir) {
+    constructor(appName, testDir, logger) {
         super();
         this.appName = appName;
         this.testDir = testDir;
+        this.logger = logger;
         this._isRunning = false;
         this.getObject = async (id) => {
             if (!this._objectsClient) {
@@ -192,7 +185,7 @@ class DBConnection extends events_1.default {
                 noFileCache: false,
                 connectTimeout: 2000,
             },
-            logger,
+            logger: this.logger,
         };
         const objectsDbPath = require.resolve(`@iobroker/db-objects-${objectsType}`, {
             paths: [
@@ -240,7 +233,7 @@ class DBConnection extends events_1.default {
                     retry_max_delay: 15000,
                 },
             },
-            logger,
+            logger: this.logger,
         };
         const statesDbPath = require.resolve(`@iobroker/db-states-${statesType}`, {
             paths: [

package/build/tests/integration/lib/harness.d.ts

@@ -21,7 +21,7 @@ export declare class TestHarness extends EventEmitter {
      * @param testDir The directory the integration tests are executed in
      */
     constructor(adapterDir: string, testDir: string, dbConnection: DBConnection);
-    private adapterName;
+    readonly adapterName: string;
     private appName;
     private testControllerDir;
     private testAdapterDir;
@@ -48,9 +48,10 @@ export declare class TestHarness extends EventEmitter {
     startAdapter(env?: NodeJS.ProcessEnv): Promise<void>;
     /**
      * Starts the adapter in a separate process and resolves after it has started
+     * @param waitForConnection By default, the test will wait for the adapter's `alive` state to become true. Set this to `true` to wait for the `info.connection` state instead.
      * @param env Additional environment variables to set
      */
-    startAdapterAndWait(env?: NodeJS.ProcessEnv): Promise<void>;
+    startAdapterAndWait(waitForConnection?: boolean, env?: NodeJS.ProcessEnv): Promise<void>;
     /** Tests if the adapter process is still running */
     isAdapterRunning(): boolean;
     /** Tests if the adapter process has already exited */

package/build/tests/integration/lib/harness.js

@@ -27,6 +27,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestHarness = void 0;
+/* eslint-disable @typescript-eslint/no-inferrable-types */
 const async_1 = require("alcalzone-shared/async");
 const objects_1 = require("alcalzone-shared/objects");
 const child_process_1 = require("child_process");
@@ -157,14 +158,16 @@ class TestHarness extends events_1.EventEmitter {
     }
     /**
      * Starts the adapter in a separate process and resolves after it has started
+     * @param waitForConnection By default, the test will wait for the adapter's `alive` state to become true. Set this to `true` to wait for the `info.connection` state instead.
      * @param env Additional environment variables to set
      */
-    async startAdapterAndWait(env = {}) {
+    async startAdapterAndWait(waitForConnection = false, env = {}) {
         return new Promise((resolve, reject) => {
+            const waitForStateId = waitForConnection
+                ? `${this.adapterName}.0.info.connection`
+                : `system.adapter.${this.adapterName}.0.alive`;
             this.on("stateChange", async (id, state) => {
-                if (id === `system.adapter.${this.adapterName}.0.alive` &&
-                    state &&
-                    state.val === true) {
+                if (id === waitForStateId && state && state.val === true) {
                     resolve();
                 }
             })

package/build/tests/integration/lib/logger.d.ts

@@ -0,0 +1,2 @@
+/// <reference types="iobroker" />
+export declare function createLogger(loglevel: ioBroker.LogLevel): ioBroker.Logger;

package/build/tests/integration/lib/logger.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLogger = void 0;
+var LoglevelOrder;
+(function (LoglevelOrder) {
+    LoglevelOrder[LoglevelOrder["error"] = 0] = "error";
+    LoglevelOrder[LoglevelOrder["warn"] = 1] = "warn";
+    LoglevelOrder[LoglevelOrder["info"] = 2] = "info";
+    LoglevelOrder[LoglevelOrder["debug"] = 3] = "debug";
+    LoglevelOrder[LoglevelOrder["silly"] = 4] = "silly";
+})(LoglevelOrder || (LoglevelOrder = {}));
+function createLogger(loglevel) {
+    var _a;
+    const loglevelNumeric = (_a = LoglevelOrder[loglevel !== null && loglevel !== void 0 ? loglevel : "debug"]) !== null && _a !== void 0 ? _a : LoglevelOrder.debug;
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    const ignore = () => { };
+    return {
+        error: loglevelNumeric >= LoglevelOrder.error ? console.error : ignore,
+        warn: loglevelNumeric >= LoglevelOrder.warn ? console.warn : ignore,
+        info: loglevelNumeric >= LoglevelOrder.info ? console.log : ignore,
+        debug: loglevelNumeric >= LoglevelOrder.debug ? console.log : ignore,
+        silly: loglevelNumeric >= LoglevelOrder.silly ? console.log : ignore,
+        level: loglevel,
+    };
+}
+exports.createLogger = createLogger;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iobroker/testing",
-  "version": "2.5.6",
+  "version": "3.0.1",
   "description": "Shared utilities for adapter and module testing in ioBroker",
   "main": "build/index.js",
   "types": "build/index.d.ts",
@@ -38,37 +38,37 @@
   },
   "homepage": "https://github.com/AlCalzone/testing#readme",
   "devDependencies": {
-    "@alcalzone/release-script": "^3.5.4",
-    "@alcalzone/release-script-plugin-license": "^3.5.3",
+    "@alcalzone/release-script": "^3.5.9",
+    "@alcalzone/release-script-plugin-license": "^3.5.9",
     "@iobroker/adapter-core": "^2.6.0",
     "@tsconfig/node12": "^1.0.9",
-    "@types/chai": "^4.3.0",
+    "@types/chai": "^4.3.1",
     "@types/chai-as-promised": "^7.1.5",
     "@types/debug": "4.1.7",
     "@types/fs-extra": "^9.0.13",
-    "@types/iobroker": "^4.0.1",
-    "@types/mocha": "^9.1.0",
-    "@types/node": "^16.11.26",
+    "@types/iobroker": "^4.0.2",
+    "@types/mocha": "^9.1.1",
+    "@types/node": "^12.20.50",
     "@types/sinon": "^10.0.11",
     "@types/sinon-chai": "^3.2.8",
-    "@typescript-eslint/eslint-plugin": "^5.13.0",
-    "@typescript-eslint/parser": "^5.13.0",
-    "eslint": "^8.10.0",
+    "@typescript-eslint/eslint-plugin": "^5.22.0",
+    "@typescript-eslint/parser": "^5.22.0",
+    "eslint": "^8.15.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.0.0",
-    "prettier": "^2.5.1",
+    "prettier": "^2.6.2",
     "rimraf": "^3.0.2",
     "source-map-support": "^0.5.21",
-    "ts-node": "^10.6.0",
-    "typescript": "~4.6.2"
+    "ts-node": "^10.7.0",
+    "typescript": "~4.6.4"
   },
   "dependencies": {
     "alcalzone-shared": "~4.0.1",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "debug": "^4.3.3",
-    "fs-extra": "^10.0.1",
-    "mocha": "^9.2.1",
+    "debug": "^4.3.4",
+    "fs-extra": "^10.1.0",
+    "mocha": "^9.2.2",
     "sinon": "^13.0.1",
     "sinon-chai": "^3.7.0"
   }