@teamscale/coverage-collector 0.1.0-beta.7 → 1.0.0-beta.2

package/dist/package.json

@@ -1,16 +1,13 @@
 {
     "name": "@teamscale/coverage-collector",
-    "version": "0.1.0-beta.7",
+    "version": "1.0.0-beta.2",
     "description": "Collector for JavaScript code coverage information",
     "main": "dist/src/main.js",
     "bin": "dist/src/main.js",
     "types": "dist/src/main.d.ts",
     "author": "CQSE GmbH",
     "license": "Apache-2.0",
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/cqse/teamscale-javascript-profiler.git"
-    },
+    "homepage": "https://docs.teamscale.com/howto/setting-up-profiler-tga/javascript/",
     "scripts": {
         "prepublishOnly": "pnpm clean && pnpm build",
         "clean": "rimraf dist tsconfig.tsbuildinfo",
@@ -22,41 +19,42 @@
         "dist/**/*"
     ],
     "dependencies": {
-        "@cqse/commons": "0.1.0-beta.6",
+        "@cqse/commons": "workspace:../cqse-commons",
         "argparse": "^2.0.1",
-        "async": "^3.2.5",
-        "axios": "^1.6.7",
+        "async": "^3.2.6",
+        "axios": "^1.7.9",
         "bunyan": "^1.8.15",
-        "date-and-time": "^3.1.1",
-        "dotenv": "^16.4.5",
-        "express": "^4.18.3",
-        "form-data": "^4.0.0",
+        "date-and-time": "^3.6.0",
+        "dotenv": "^16.4.7",
+        "express": "^5.0.1",
+        "form-data": "^4.0.2",
         "mkdirp": "^3.0.1",
+        "node-cache": "^5.1.2",
         "rxjs": "^7.8.1",
         "source-map": "^0.7.4",
         "tmp": "^0.2.3",
         "typescript-optional": "^2.0.1",
-        "ws": "^8.16.0"
+        "ws": "^8.18.0"
     },
     "devDependencies": {
-        "@babel/core": "^7.24.0",
-        "@babel/preset-env": "^7.24.0",
-        "@types/argparse": "^2.0.15",
+        "@babel/core": "^7.26.8",
+        "@babel/preset-env": "^7.26.8",
+        "@types/argparse": "^2.0.17",
         "@types/async": "^3.2.24",
         "@types/bunyan": "^1.8.11",
-        "@types/express": "^4.17.21",
-        "@types/jest": "^29.5.12",
-        "@types/node": "^20.11.28",
+        "@types/express": "^5.0.0",
+        "@types/jest": "^29.5.14",
+        "@types/node": "^22.13.4",
         "@types/tmp": "^0.2.6",
-        "@types/ws": "^8.5.10",
+        "@types/ws": "^8.5.14",
         "babel-jest": "^29.7.0",
-        "esbuild": "^0.20.2",
+        "esbuild": "^0.25.0",
         "jest": "^29.7.0",
-        "mockttp": "^3.10.1",
-        "rimraf": "^5.0.5",
-        "ts-jest": "^29.1.2",
+        "mockttp": "3.15.5",
+        "rimraf": "^6.0.1",
+        "ts-jest": "^29.2.5",
         "ts-node": "^10.9.2",
-        "typescript": "^5.4.2"
+        "typescript": "^5.7.3"
     },
     "publishConfig": {
         "access": "public"

package/dist/src/config/RemoteProfilerConfig.d.ts

@@ -0,0 +1,24 @@
+import { CollectorOptions } from '@cqse/commons';
+import Logger from 'bunyan';
+type ProfilerConfiguration = {
+    configurationId: string;
+    configurationOptions?: string;
+};
+/**
+ * Utility methods for handling remote profiler configuration. The configuration is retrieved from Teasmcale.
+ */
+export declare class RemoteProfilerConfig {
+    /**
+     * Query the profiler configuration with the given ID via the Teamscale API.
+     */
+    static queryConfiguration(baseConfig: CollectorOptions, configId: string, logger: Logger): Promise<Map<string, string>>;
+    /**
+     * Visible for testing.
+     */
+    static parseConfigurationStringIntoMap(response: ProfilerConfiguration): Map<string, string>;
+}
+/**
+ * For testing, clear the configuration cache.
+ */
+export declare function clearConfigurationCache(): void;
+export {};

package/dist/src/config/RemoteProfilerConfig.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteProfilerConfig = void 0;
+exports.clearConfigurationCache = clearConfigurationCache;
+const RestApis_1 = require("../utils/RestApis");
+const commons_1 = require("@cqse/commons");
+const axios_1 = __importDefault(require("axios"));
+const node_cache_1 = __importDefault(require("node-cache"));
+/**
+ * We cache the configuration we got from Teamscale for 60s.
+ */
+const CONFIG_CACHE = new node_cache_1.default({ stdTTL: 60, deleteOnExpire: true });
+/**
+ * Utility methods for handling remote profiler configuration. The configuration is retrieved from Teasmcale.
+ */
+class RemoteProfilerConfig {
+    /**
+     * Query the profiler configuration with the given ID via the Teamscale API.
+     */
+    static async queryConfiguration(baseConfig, configId, logger) {
+        if (!baseConfig.teamscaleServerUrl || !baseConfig.teamscaleAccessToken || !baseConfig.teamscaleUser) {
+            throw new Error("Access to Teamscale is not configured. Receiving profiler configurations is not possible. " +
+                "Please specify configuration arguments for --teamscale-server-url, --teamscale-user, and --teamscale-access-token");
+        }
+        let result = CONFIG_CACHE.get(configId);
+        try {
+            if (result) {
+                logger.debug('Using cached configuration with id ' + configId);
+                return result;
+            }
+            logger.debug(`Requesting configuration with ID ${configId} from Teamscale.`);
+            const response = await (0, RestApis_1.performRequest)(`${(0, RestApis_1.removeTrailingUrlSlash)(baseConfig.teamscaleServerUrl)}/api/${RestApis_1.TEAMSCALE_API_VERSION}/profiler-configurations/${configId}`, undefined, (0, RestApis_1.prepareTeamscaleApiRequestConfig)(baseConfig, {}), axios_1.default.get, logger);
+            if (response === undefined) {
+                throw new commons_1.InvalidConfigurationException(`No configuration found with ID ${configId}`);
+            }
+            // Fail for an empty configuration. We do not accept an empty configuration
+            // for a specified configuration ID.
+            if (!response.configurationOptions || response.configurationOptions.trim().length === 0) {
+                throw new commons_1.InvalidConfigurationException(`Configuration with ID "${configId}" is empty. Please specify relevant configuration options.`);
+            }
+            result = this.parseConfigurationStringIntoMap(response);
+            CONFIG_CACHE.set(configId, result);
+        }
+        catch (error) {
+            throw new Error(`Failed to retrieve configuration with ID ${configId}: ${error.message}`, { cause: error });
+        }
+        return result;
+    }
+    /**
+     * Visible for testing.
+     */
+    static parseConfigurationStringIntoMap(response) {
+        const result = new Map();
+        (response.configurationOptions ?? "").split(/[\r\n]+/).map(line => line.trim())
+            .filter(line => line.length > 0)
+            .filter(line => !line.startsWith('#'))
+            .forEach(line => {
+            const split = line.split('=');
+            if (split.length !== 2) {
+                throw new commons_1.InvalidConfigurationException(`Invalid configuration line; expecting a valid key=value pair: ${line}`);
+            }
+            result.set(split[0].trim(), split[1].trim());
+        });
+        return result;
+    }
+}
+exports.RemoteProfilerConfig = RemoteProfilerConfig;
+/**
+ * For testing, clear the configuration cache.
+ */
+function clearConfigurationCache() {
+    CONFIG_CACHE.flushAll();
+}

package/dist/src/control/App.d.ts

@@ -0,0 +1,31 @@
+import { ConfigurationParameters, StaticCollectorOptions } from '@cqse/commons';
+import 'dotenv/config';
+/**
+ * Callback interface to be called to stop a process in a controlled fashion.
+ */
+export type Stoppable = {
+    stop: () => Promise<void>;
+};
+/**
+ * The main class of the Teamscale JavaScript Collector.
+ * Used to start the collector with a given configuration.
+ */
+export declare class App {
+    /**
+     * Construct the logger.
+     */
+    private static buildLogger;
+    /**
+     * Entry point of the Teamscale JavaScript Profiler.
+     */
+    static run(): Promise<Stoppable>;
+    /**
+     * Run the collector with the given configuration options.
+     */
+    static runWithConfig(staticParameters: ConfigurationParameters, config: StaticCollectorOptions): Promise<Stoppable>;
+    /**
+     * Starts a timer that shows a message every min that no coverage
+     * was received until the opposite is the case.
+     */
+    private static startNoMessageTimer;
+}

package/dist/src/control/App.js

@@ -0,0 +1,202 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.App = void 0;
+const CoverageDumper_1 = require("./CoverageDumper");
+const StdConsoleLogger_1 = require("../utils/StdConsoleLogger");
+const PrettyFileLogger_1 = require("../utils/PrettyFileLogger");
+const DataStorage_1 = require("../storage/DataStorage");
+const CollectingServer_1 = require("../receiver/CollectingServer");
+const ControlServer_1 = require("./ControlServer");
+const commons_1 = require("@cqse/commons");
+const package_json_1 = require("../../package.json");
+const bunyan_1 = __importDefault(require("bunyan"));
+require("dotenv/config");
+const fs = __importStar(require("fs"));
+const mkdirp_1 = require("mkdirp");
+const path_1 = __importDefault(require("path"));
+const TeamscaleUpload_1 = require("../upload/TeamscaleUpload");
+const process = __importStar(require("node:process"));
+/**
+ * The main class of the Teamscale JavaScript Collector.
+ * Used to start the collector with a given configuration.
+ */
+class App {
+    /**
+     * Construct the logger.
+     */
+    static buildLogger(config) {
+        const logfilePath = config.logToFile.trim();
+        mkdirp_1.mkdirp.sync(path_1.default.dirname(logfilePath));
+        const logLevel = config.logLevel;
+        const logger = bunyan_1.default.createLogger({
+            name: 'Collector',
+            streams: [
+                // console output
+                { level: logLevel, stream: new StdConsoleLogger_1.StdConsoleLogger(), type: 'raw' },
+                // default log file
+                { level: logLevel, stream: new PrettyFileLogger_1.PrettyFileLogger(fs.createWriteStream(logfilePath)), type: 'raw' }
+            ]
+        });
+        // If the given flag is set, we also log with a JSON-like format
+        if (config.jsonLog) {
+            logger.addStream({ level: logLevel, path: `${logfilePath}.json` });
+        }
+        return logger;
+    }
+    /**
+     * Entry point of the Teamscale JavaScript Profiler.
+     */
+    static async run() {
+        const configParameters = (0, commons_1.buildStaticCollectorParameters)();
+        const appInfos = { about: package_json_1.description, version: package_json_1.version, name: package_json_1.name };
+        const config = (0, commons_1.processCommandLine)(configParameters, appInfos);
+        return await App.runWithConfig(configParameters, config);
+    }
+    /**
+     * Run the collector with the given configuration options.
+     */
+    static async runWithConfig(staticParameters, config) {
+        // Build the logger
+        const logger = this.buildLogger(config);
+        logger.info(`Starting collector in working directory "${process.cwd()}".`);
+        logger.info(`Will dump coverage to directory "${config.dumpFolder}".`);
+        logger.info(`Logging "${config.logLevel}" to "${config.logToFile}".`);
+        // Check the connection to Teamscale, if needed.
+        if (config.teamscaleServerUrl) {
+            const checkResult = await (0, TeamscaleUpload_1.checkTeamscaleCredentials)(config, logger);
+            if (!checkResult) {
+                logger.error('Could not connect to Teamscale with the given credential. Please check your configuration.');
+                process.exit(1);
+            }
+        }
+        // Ensure that the root coverage folder is writable.
+        if (config.dumpFolder) {
+            const dumpFolder = path_1.default.resolve(config.dumpFolder);
+            ensureWritableFolder(dumpFolder, logger);
+        }
+        // Prepare the storage and the server
+        const reconfigurableParameters = (0, commons_1.buildReconfigurableCollectorParameters)();
+        // Now also add checks for configuration arguments that are to be online, that is,
+        // when receiving configuration updates from Teamscale or via the control API.
+        reconfigurableParameters.addArgumentCheck((options) => {
+            if (config.teamscaleServerUrl) {
+                // While the connection to Teamscale can be configured, not all
+                // uploads might be targeted to that but some shall still be stored to the disk.
+                // In case a Teamscale project is configured, we expect that an upload to Teamscale shall happen.
+                if (options.teamscaleProject && !options.teamscalePartition) {
+                    return 'The Teamscale project (parameter teamscaleProject) and coverage partition (parameter teamscalePartition) ' +
+                        'must be configured for an upload to Teamscale.';
+                }
+            }
+        });
+        const storage = new DataStorage_1.DataStorage(logger, (0, commons_1.parameterUnion)(staticParameters, reconfigurableParameters), reconfigurableParameters, config);
+        const server = new CollectingServer_1.WebSocketCollectingServer(config.port, storage, logger);
+        // Enable the remote control API if configured
+        const controlServer = new ControlServer_1.ControlServer(config, storage, logger);
+        const controlServerState = controlServer.start();
+        // Start the server socket.
+        // ATTENTION: The server is executed asynchronously.
+        const serverState = server.start();
+        // Optionally, start a timer that dumps the coverage after N seconds
+        const dumpTimerState = CoverageDumper_1.CoverageDumper.startRegularCollectorProcesses(storage, logger);
+        // Start a timer that informs if no coverage was received within the last minute
+        const statsTimerState = this.startNoMessageTimer(logger, server);
+        const stop = async function () {
+            logger.info('Stopping the collector.');
+            // Final dump before stop. The await/async construct that we use here
+            // is used to make sure that other events in the event loop are processed
+            // before the actual dump happens, that is, to retrieve coverage that was
+            // already sent but has not yet been processed in the collector.
+            // We need this, for example, in our system tests where everything runs in one NodeJS environment.
+            await new Promise(resolve => {
+                setTimeout(async () => {
+                    await CoverageDumper_1.CoverageDumper.dumpCoverage(storage, logger);
+                    resolve(undefined);
+                }, 0);
+            });
+            // Stop all timers and sockets.
+            dumpTimerState.stop();
+            statsTimerState.stop();
+            await controlServerState.stop();
+            serverState.stop();
+            logger.info('Bye bye.');
+        };
+        return { stop };
+    }
+    /**
+     * Starts a timer that shows a message every min that no coverage
+     * was received until the opposite is the case.
+     */
+    static startNoMessageTimer(logger, server) {
+        const startTime = Date.now();
+        const timer = setInterval(async () => {
+            const stats = server.getStatistics();
+            if (stats.totalCoverageMessages === 0) {
+                logger.info(`No coverage received for ${((Date.now() - startTime) / 1000.0).toFixed(0)}s.`);
+            }
+            else {
+                // We can stop running the timer after we have received the first coverage.
+                clearInterval(timer);
+            }
+        }, 1000 * 60);
+        return {
+            stop: () => clearInterval(timer)
+        };
+    }
+}
+exports.App = App;
+function ensureWritableFolder(dumpFolder, logger) {
+    try {
+        // 1. Create the folder when it doesn't exist (recursive = true is a no-op when it does).
+        mkdirp_1.mkdirp.sync(dumpFolder);
+        // 2. Verify that we ended up with a directory.
+        const stat = fs.statSync(dumpFolder);
+        if (!stat.isDirectory()) {
+            throw new Error(`"${dumpFolder}" exists but is not a directory.`);
+        }
+        // 3. Perform a real write test.
+        const testFile = path_1.default.join(dumpFolder, `.write-test-${Date.now()}`);
+        fs.writeFileSync(testFile, 'writable?');
+        fs.unlinkSync(testFile);
+    }
+    catch (err) {
+        logger.error(`The configured dump folder "${dumpFolder}" is not writable or usable: ${err.message}`);
+        process.exit(1);
+    }
+}

package/dist/src/control/ControlServer.d.ts

@@ -0,0 +1,24 @@
+import { DataStorage } from '../storage/DataStorage';
+import Logger from 'bunyan';
+import { CollectorOptions } from '@cqse/commons';
+/**
+ * Provides a REST API for remote configuration of the collector.
+ */
+export declare class ControlServer {
+    private config;
+    private storage;
+    private logger;
+    constructor(config: CollectorOptions, storage: DataStorage, logger: Logger);
+    /**
+     * Start the collector remote config API.
+     */
+    start(): {
+        stop: () => Promise<void>;
+    };
+    private handleRefreshConfigs;
+    private handleDumpPost;
+    private handleDumpPostForConfig;
+    private handleGlobalCoverageReset;
+    private handleCoverageResetForConfig;
+    private handleConfigScopedRequest;
+}

package/dist/src/control/ControlServer.js

@@ -0,0 +1,100 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ControlServer = void 0;
+const CoverageDumper_1 = require("../control/CoverageDumper");
+const express_1 = __importDefault(require("express"));
+/**
+ * Provides a REST API for remote configuration of the collector.
+ */
+class ControlServer {
+    config;
+    storage;
+    logger;
+    constructor(config, storage, logger) {
+        this.config = config;
+        this.storage = storage;
+        this.logger = logger;
+    }
+    /**
+     * Start the collector remote config API.
+     */
+    start() {
+        if (!this.config.enableControlPort) {
+            return {
+                async stop() {
+                    // nothing to stop in this case
+                }
+            };
+        }
+        const controlServer = (0, express_1.default)();
+        controlServer.use(express_1.default.text({}));
+        controlServer.use(express_1.default.urlencoded({ extended: true }));
+        const serverSocket = controlServer.listen(this.config.enableControlPort);
+        controlServer.post('/refresh/', (request, response) => this.handleRefreshConfigs(request, response));
+        controlServer.post('/dump/', (request, response) => this.handleDumpPost(request, response));
+        controlServer.post('/dump/:configId', (request, response) => this.handleDumpPostForConfig(request, response));
+        controlServer.post('/reset', (request, response) => this.handleGlobalCoverageReset(request, response));
+        controlServer.post('/reset/:configId', (request, response) => this.handleCoverageResetForConfig(request, response));
+        this.logger.info(`Control server enabled at port ${this.config.enableControlPort}.`);
+        return {
+            async stop() {
+                return new Promise(resolve => {
+                    serverSocket.close(() => resolve());
+                });
+            }
+        };
+    }
+    async handleRefreshConfigs(request, response) {
+        this.logger.info('Remote configuration refresh requested via the control API.');
+        await this.storage.refreshAllRemoteConfigurations();
+        this.logger.info('Refresh done.');
+        response.sendStatus(200);
+    }
+    async handleDumpPost(request, response) {
+        this.logger.info('Dumping coverage requested via the control API.');
+        await CoverageDumper_1.CoverageDumper.dumpCoverage(this.storage, this.logger);
+        response.sendStatus(200);
+    }
+    async handleDumpPostForConfig(request, response) {
+        return this.handleConfigScopedRequest(request, response, async (configId) => {
+            this.logger.info(`Dumping coverage requested for config '${configId}' via the control API.`);
+            const appsWithConfig = this.storage.getApplicationsWithConfig(configId);
+            for (const appId of appsWithConfig) {
+                await CoverageDumper_1.CoverageDumper.dumpCoverage(this.storage, this.logger, appId);
+            }
+        });
+    }
+    async handleGlobalCoverageReset(request, response) {
+        this.storage.discardCollectedCoverage();
+        this.logger.info(`Discarding collected coverage information as requested via the control API.`);
+        response.sendStatus(200);
+    }
+    async handleCoverageResetForConfig(request, response) {
+        return this.handleConfigScopedRequest(request, response, async (configId) => {
+            this.logger.info(`Discarding collected coverage information for config '${configId}' as requested via the control API.`);
+            const appsWithConfig = this.storage.getApplicationsWithConfig(configId);
+            for (const appId of appsWithConfig) {
+                this.storage.discardCollectedCoverage(appId);
+            }
+        });
+    }
+    async handleConfigScopedRequest(request, response, configHandler) {
+        const configId = request.params.configId;
+        if (!configId) {
+            throw new Error('Invalid config ID');
+        }
+        try {
+            await configHandler(configId);
+            response.sendStatus(200);
+        }
+        catch (error) {
+            response.set('Content-Type', 'text/plain');
+            response.send(`Failed to handle request for config '${configId}': ${error.message}`);
+            response.sendStatus(500);
+        }
+    }
+}
+exports.ControlServer = ControlServer;

package/dist/src/control/CoverageDumper.d.ts

@@ -0,0 +1,27 @@
+import { DataStorage } from '../storage/DataStorage';
+import Logger from 'bunyan';
+/**
+ * Functionality for dumping coverage regularly to different targets.
+ */
+export declare class CoverageDumper {
+    /**
+     * Start a timer for dumping the data, depending on the configuration;
+     * and for retrieving new configuration values from Teamscale.
+     *
+     * @param storage - The storage with the information to dump.
+     * @param logger - The logger to use.
+     */
+    static startRegularCollectorProcesses(storage: DataStorage, logger: Logger): {
+        stop: () => void;
+    };
+    /**
+     * Dump all the collected coverage. If an `onlyForAppId` is given, only for the app with that ID.
+     * If no `onlyForAppId` is given, coverage will be dumped for every application, separately.
+     */
+    static dumpCoverage(storage: DataStorage, logger: Logger, onlyForAppId?: string): Promise<void>;
+    /**
+     * Returns true if an actual upload request was sent to the remote server.
+     */
+    private static uploadCoverage;
+    private static determineCoverageTargetFolder;
+}