@teamscale/coverage-collector 0.1.0-beta.8 → 1.0.0-beta.2

package/dist/src/control/CoverageDumper.js

@@ -0,0 +1,166 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CoverageDumper = void 0;
+const RestApis_1 = require("../utils/RestApis");
+const TeamscaleUpload_1 = require("../upload/TeamscaleUpload");
+const ArtifactoryUpload_1 = require("../upload/ArtifactoryUpload");
+const fs_1 = __importDefault(require("fs"));
+const CONFIG_REQUERY_INTERVAL_SECONDS = 53;
+const DUMP_COVERAGE_CHECK_AFTER_SECONDS = 60;
+const CHECK_FOR_SCHEDULED_ACTIVITY_AFTER_SECONDS = 31;
+const DEFAULT_COVERAGE_DUMP_AFTER_MINS = 10;
+const STARTUP_TIMESTAMP = Date.now();
+/**
+ * Functionality for dumping coverage regularly to different targets.
+ */
+class CoverageDumper {
+    /**
+     * Start a timer for dumping the data, depending on the configuration;
+     * and for retrieving new configuration values from Teamscale.
+     *
+     * @param storage - The storage with the information to dump.
+     * @param logger - The logger to use.
+     */
+    static startRegularCollectorProcesses(storage, logger) {
+        const lastConfigRefreshPerApp = new Map();
+        const lastDumpTimestampPerApp = new Map();
+        const doIfExpired = async (key, lastExecutionTimestamps, doAfterMillis, action) => {
+            const lastExecutionTimestamp = lastExecutionTimestamps.get(key) ?? STARTUP_TIMESTAMP;
+            if (Date.now() - lastExecutionTimestamp > doAfterMillis) {
+                try {
+                    await action();
+                }
+                catch (error) {
+                    logger.error(`Regular collector action failed with an error: ` + error?.message);
+                    logger.error(error);
+                }
+                lastExecutionTimestamps.set(key, Date.now());
+            }
+        };
+        const timer = setInterval(async () => {
+            // 1. Re-query the profiler configurations (can be more than one,
+            // one for each application) once every minute.
+            for (const appId of storage.getApplicationIDs()) {
+                await doIfExpired(`config-update-${appId}`, lastConfigRefreshPerApp, 1000 * CONFIG_REQUERY_INTERVAL_SECONDS, async () => {
+                    await storage.refreshApplicationConfiguration(appId);
+                });
+            }
+            // 2. For each application, check if coverage shall be dumped once a minute.
+            // Each application can be configured differently, even if the same config id was assigned initially.
+            for (const appId of storage.getApplicationIDs()) {
+                const appDumpAfterMinutes = storage.getAppConfiguration(appId).dumpAfterMins ?? DEFAULT_COVERAGE_DUMP_AFTER_MINS;
+                await doIfExpired(`coverage-dump-${appId}`, lastDumpTimestampPerApp, 1000 * DUMP_COVERAGE_CHECK_AFTER_SECONDS * appDumpAfterMinutes, async () => {
+                    await this.dumpCoverage(storage, logger, appId);
+                });
+            }
+        }, 1000 * CHECK_FOR_SCHEDULED_ACTIVITY_AFTER_SECONDS);
+        // Dump the coverage before each (correct) config update.
+        storage.addBeforeConfigUpdateCallback(async (appId, dataStorage) => {
+            logger.info(`Dumping coverage before config update for application ${appId}.`);
+            await this.dumpCoverage(dataStorage, logger, appId);
+        });
+        logger.info(`Started the regular remote configuration refresh and coverage dump.`);
+        return {
+            stop: () => {
+                if (timer) {
+                    clearInterval(timer);
+                }
+            }
+        };
+    }
+    /**
+     * Dump all the collected coverage. If an `onlyForAppId` is given, only for the app with that ID.
+     * If no `onlyForAppId` is given, coverage will be dumped for every application, separately.
+     */
+    static async dumpCoverage(storage, logger, onlyForAppId) {
+        const appsToDumpFor = onlyForAppId ? [onlyForAppId] : storage.getApplicationIDs();
+        const dumpTimestamp = new Date();
+        let hadCoverageToDump = false;
+        let hadUploadSent = false;
+        for (const dumpForAppId of appsToDumpFor) {
+            const dumpConfiguration = storage.getAppConfiguration(dumpForAppId);
+            const dumpToFolder = this.determineCoverageTargetFolder(dumpForAppId, dumpConfiguration);
+            const deleteAfterUpload = !(dumpConfiguration.keepCoverageFiles ?? dumpConfiguration.dumpToFolder !== undefined);
+            try {
+                // Write coverage to a file
+                const dumpSummary = storage.dumpToSimpleCoverageFile(dumpToFolder, dumpTimestamp, dumpForAppId);
+                hadCoverageToDump = dumpSummary.hadCoverageToDump || hadCoverageToDump;
+                // Upload to Teamscale or Artifactory if configured
+                for (const { simpleCoverageFile, simpleCoverageFileLines, commit } of dumpSummary.details) {
+                    try {
+                        if (dumpConfiguration.teamscaleServerUrl || dumpConfiguration.artifactoryServerUrl) {
+                            const coverageUploaded = await this.uploadCoverage(dumpConfiguration, simpleCoverageFile, simpleCoverageFileLines, commit, logger, deleteAfterUpload);
+                            hadUploadSent = coverageUploaded || hadUploadSent;
+                        }
+                    }
+                    catch (e) {
+                        if (e instanceof RestApis_1.UploadError) {
+                            logger.error(`Coverage upload failed for application ${dumpForAppId} and commit ${commit}.\n`
+                                + `The coverage files on disk (inside the folder "${dumpToFolder}") were not deleted. `
+                                + `You can still upload them manually. Error: ${e.message}`);
+                        }
+                        else {
+                            logger.error('Coverage dump failed', e);
+                        }
+                    }
+                }
+            }
+            catch (e) {
+                logger.error(`Dumping coverage failed for application ${dumpForAppId}`, e);
+            }
+        }
+        if (hadUploadSent) {
+            logger.info(`Uploaded coverage for timestamp ${dumpTimestamp.toISOString()} to a remote server.`);
+        }
+        if (!hadCoverageToDump) {
+            logger.info('Coverage dump request was processed successfully; no new coverage to dump found.');
+        }
+    }
+    /**
+     * Returns true if an actual upload request was sent to the remote server.
+     */
+    static async uploadCoverage(config, coverageFile, lines, commit, logger, deleteAfterUpload) {
+        let uploadDone = false;
+        if (config.teamscaleServerUrl && (config.teamscaleProject ?? "").length > 0) {
+            // A Teamscale url might be configured for remote configuration. Uploads might not be intended.
+            uploadDone = await (0, TeamscaleUpload_1.uploadToTeamscale)(config, logger, coverageFile, lines, commit);
+        }
+        if (config.artifactoryServerUrl) {
+            uploadDone = uploadDone || await (0, ArtifactoryUpload_1.uploadToArtifactory)(config, logger, coverageFile, lines, commit);
+        }
+        // Delete coverage if upload was successful and keeping coverage files on disk was not configure by the user
+        if (deleteAfterUpload) {
+            fs_1.default.unlinkSync(coverageFile);
+        }
+        return uploadDone;
+    }
+    static determineCoverageTargetFolder(appId, config) {
+        const result = `${config.dumpFolder}/${makeProperDirectoryPath(config.dumpToFolder)}${appId}`;
+        fs_1.default.mkdirSync(result, { recursive: true });
+        return result;
+    }
+}
+exports.CoverageDumper = CoverageDumper;
+/**
+ * Only characters that allowed in most file systems. And replace backslashes with forward slashes.
+ */
+function makeProperDirectoryPath(path) {
+    if (path === undefined || path.trim().length === 0) {
+        return '';
+    }
+    // Remove leading slashes.
+    path = path.trim().replace(/^\/+/g, '');
+    // Replace backslashes with forward slashes.
+    path = path.replace(/\\/g, '/');
+    // Remove all characters that are not allowed in file paths.
+    // Important: We do not allow points in file paths to avoid directory traversal attacks/confusion.
+    path = path.replace(/[^a-zA-Z0-9\-_/]/g, '_');
+    // Ensure a trailing slash.
+    if (path.endsWith('/')) {
+        return path;
+    }
+    return `${path}/`;
+}

package/dist/src/main.js

@@ -1,5 +1,32 @@
 #!/usr/bin/env node
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-const App_1 = require("./App");
-App_1.App.run();
+const App_1 = require("./control/App");
+const node_process_1 = __importDefault(require("node:process"));
+const commons_1 = require("@cqse/commons");
+/**
+ * Start the application from its entry point function `App.run()`.
+ * The present file is defined to be the main file of the collector in `package.json`.
+ */
+(async () => {
+    const app = await App_1.App.run();
+    const handleExit = async () => {
+        await app.stop();
+        node_process_1.default.exit();
+    };
+    // Graceful termination on CTRL+C and SIGTERM.
+    node_process_1.default.on('SIGINT', () => handleExit());
+    node_process_1.default.on('SIGTERM', () => handleExit());
+})().catch(reason => {
+    if (reason instanceof commons_1.InvalidConfigurationException) {
+        // Do not print the stack trace for invalid configuration parameters from the user.
+        console.error(`Failed: ${reason.message}`);
+    }
+    else {
+        console.error('Failed: ', reason);
+    }
+    node_process_1.default.exit(1);
+});

package/dist/src/receiver/CollectingServer.d.ts

@@ -1,4 +1,4 @@
-import { IDataStorage } from '../storage/DataStorage';
+import { DataStorage } from '../storage/DataStorage';
 import Logger from 'bunyan';
 /**
  * A WebSocket based implementation of a coverage receiver.
@@ -32,7 +32,7 @@ export declare class WebSocketCollectingServer {
      * @param storage - The storage to put the received coverage information to.
      * @param logger - The logger to use.
      */
-    constructor(port: number, storage: IDataStorage, logger: Logger);
+    constructor(port: number, storage: DataStorage, logger: Logger);
     /**
      * Start the server socket, handle sessions and dispatch messages.
      */

package/dist/src/receiver/CollectingServer.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebSocketCollectingServer = void 0;
 const WebSocket = __importStar(require("ws"));
@@ -29,11 +39,33 @@ const commons_1 = require("@cqse/commons");
 const Session_1 = require("./Session");
 /** A message that provides coverage information */
 const MESSAGE_TYPE_COVERAGE = 'c';
+/** A message that defines the bucket to put the coverage into */
+const MESSAGE_TYPE_BUCKET = 'b';
 /**
  * A WebSocket based implementation of a coverage receiver.
  * Receives coverage from instrumented JavaScript code.
  */
 class WebSocketCollectingServer {
+    /**
+     * The WebSocket server component.
+     */
+    server;
+    /**
+     * The storage to put the received coverage information to for aggregation and further processing.
+     */
+    storage;
+    /**
+     * The logger to use.
+     */
+    logger;
+    /**
+     * The number of messages that have been received.
+     */
+    totalNumMessagesReceived;
+    /**
+     * The number of coverage messages that have been received.
+     */
+    totalNumCoverageMessagesReceived;
     /**
      * Constructor.
      *
@@ -53,11 +85,10 @@ class WebSocketCollectingServer {
      * Start the server socket, handle sessions and dispatch messages.
      */
     start() {
-        var _a, _b;
-        this.logger.info(`Starting server on port ${(_a = this.server) === null || _a === void 0 ? void 0 : _a.options.port}.`);
+        this.logger.info(`Starting server on port ${this.server?.options.port}.`);
         // Handle new connections from clients
-        (_b = this.server) === null || _b === void 0 ? void 0 : _b.on('connection', (webSocket, req) => {
-            let session = new Session_1.Session(req.socket, this.storage, this.logger);
+        this.server?.on('connection', (webSocket, req) => {
+            let session = new Session_1.Session(this.storage);
             this.logger.trace(`Connection from: ${req.socket.remoteAddress}`);
             // Handle disconnecting clients
             webSocket.on('close', code => {
@@ -80,8 +111,8 @@ class WebSocketCollectingServer {
         });
         return {
             stop: () => {
-                var _a;
-                (_a = this.server) === null || _a === void 0 ? void 0 : _a.close();
+                this.server?.clients.forEach(client => client.close());
+                this.server?.close();
                 this.server = null;
             }
         };
@@ -95,13 +126,26 @@ class WebSocketCollectingServer {
     async handleMessage(session, message) {
         try {
             const messageType = message.toString('utf8', 0, 1);
-            if (messageType.startsWith(MESSAGE_TYPE_COVERAGE)) {
+            if (messageType === MESSAGE_TYPE_BUCKET) {
+                const bucketDefinitionEncoded = message.toString('utf-8', 1).trim();
+                const bucketDefinitionDecoded = Buffer.from(bucketDefinitionEncoded, 'base64').toString('utf8');
+                const bucketDefinition = JSON.parse(bucketDefinitionDecoded);
+                this.logger.debug(`Received bucket definition: ${JSON.stringify(bucketDefinition)} for session ${session.sessionId}.`);
+                await session.setupBucketAndApplication(bucketDefinition);
+            }
+            else if (messageType === MESSAGE_TYPE_COVERAGE) {
                 this.totalNumCoverageMessagesReceived += 1;
-                await this.handleCoverageMessage(session, message.subarray(1));
+                const linesCoveredApprox = this.handleCoverageMessage(session, message.subarray(1));
+                if (linesCoveredApprox > 0) {
+                    this.logger.debug(`Coverage for approximately ${linesCoveredApprox} lines processed for session ${session.sessionId}.`);
+                }
+            }
+            else {
+                this.logger.error(`Unknown message type: ${messageType}`);
             }
         }
         catch (e) {
-            this.logger.error(`Error while processing message starting with ${message.toString('utf8', 0, Math.min(50, message.length))}`);
+            this.logger.error(`Error while processing message starting with ${message.toString('utf8', 0, Math.min(250, message.length))}`);
             this.logger.error(e.message);
         }
     }
@@ -121,7 +165,9 @@ class WebSocketCollectingServer {
      * newline and semicolon symbols separate tokens.
      * A file name is a token; a range (start to end line) is a token.
      */
-    async handleCoverageMessage(session, body) {
+    handleCoverageMessage(session, body) {
+        // Number of lines signaled to be covered (just an approximation, for debugging purposes)
+        let lineCoverageStatements = 0;
         // Split the input into tokens; these are either file names or code ranges.
         const tokens = body.toString().split(/[\n;]/).map(line => line.trim());
         // Placeholder for group/filename.
@@ -129,20 +175,24 @@ class WebSocketCollectingServer {
         tokens.forEach(token => {
             // Check if the token starts with '@' - indicating a new group/filename.
             if (token.startsWith('@')) {
-                filename = token.substring(1).trim(); // Remove '@' character and extra spaces.
+                filename = token.substring(1).trim(); // Remove the '@' character and extra spaces.
             }
             else if (filename) {
                 // It is not a file name, we have a range token here.
                 // Example for range a token: `3-5`, or just `42` (corresponding to `42-42`).
                 const range = token.split(/,|-/).map(value => Number.parseInt(value));
                 if (range.length === 1) {
-                    session.putLineCoverage(filename, range[0], range[0]);
+                    lineCoverageStatements += session.putLineCoverage(filename, range[0], range[0]);
                 }
                 else if (range.length === 2) {
-                    session.putLineCoverage(filename, range[0], range[1]);
+                    lineCoverageStatements += session.putLineCoverage(filename, range[0], range[1]);
+                }
+                else {
+                    this.logger.error(`Invalid range token: ${token}`);
                 }
             }
         });
+        return lineCoverageStatements;
     }
     /**
      * Returns a statistic on the number of messages received.

package/dist/src/receiver/Session.d.ts

@@ -1,18 +1,4 @@
-/// <reference types="node" />
-import { Socket } from 'net';
-import { IDataStorage } from '../storage/DataStorage';
-import Logger from 'bunyan';
-/**
- * Coverage information that has not been mapped back to the
- * original code using a source map.
- */
-export type UnmappedCoverage = {
-    fileId: string;
-    startLine: number;
-    startColumn: number;
-    endLine: number;
-    endColumn: number;
-};
+import { CoverageBucketSpecifier, DataStorage } from '../storage/DataStorage';
 /**
  * The session maintains the relevant information for a client.
  * One session is created for each client.
@@ -20,29 +6,31 @@ export type UnmappedCoverage = {
  */
 export declare class Session {
     /**
-     * The client socket.
+     * An internal session ID for debugging purposes.
      */
-    private readonly socket;
+    private readonly internalSessionId;
     /**
      * The storage to forward coverage information to for aggregation.
      */
     private readonly storage;
     /**
-     * The logger to use.
+     * The application to contribute coverage to.
      */
-    private readonly logger;
+    private appId?;
     /**
-     * The project the coverage information is for.
+     * The commit to contribute coverage for.
      */
-    private readonly projectId;
+    private commitId?;
     /**
      * Constructor
      *
-     * @param socket - The client socket.
      * @param storage - The storage to store and aggregate coverage information in.
-     * @param logger - The logger to use.
      */
-    constructor(socket: Socket, storage: IDataStorage, logger: Logger);
+    constructor(storage: DataStorage);
+    /**
+     * If not already done, the bucket to send coverage into for the given application is set up.
+     */
+    setupBucketAndApplication(bucketSpecifier: CoverageBucketSpecifier): Promise<void>;
     /**
      * Put coverage information to the storage for aggregation.
      * This method also conducts the mapping based on the source map.
@@ -51,5 +39,7 @@ export declare class Session {
      * @param startLine - The line number within the bundle the range starts.
      * @param endLine - The line number within the bundle the range ends.
      */
-    putLineCoverage(fileId: string, startLine: number, endLine: number): void;
+    putLineCoverage(fileId: string, startLine: number, endLine: number): number;
+    /** Returns the internal session ID for debugging purposes. */
+    get sessionId(): string;
 }

package/dist/src/receiver/Session.js

@@ -8,18 +8,38 @@ const commons_1 = require("@cqse/commons");
  * The mapping based on sourcemaps is conducted here.
  */
 class Session {
+    /**
+     * An internal session ID for debugging purposes.
+     */
+    internalSessionId;
+    /**
+     * The storage to forward coverage information to for aggregation.
+     */
+    storage;
+    /**
+     * The application to contribute coverage to.
+     */
+    appId;
+    /**
+     * The commit to contribute coverage for.
+     */
+    commitId;
     /**
      * Constructor
      *
-     * @param socket - The client socket.
      * @param storage - The storage to store and aggregate coverage information in.
-     * @param logger - The logger to use.
      */
-    constructor(socket, storage, logger) {
-        this.socket = commons_1.Contract.requireDefined(socket);
+    constructor(storage) {
         this.storage = commons_1.Contract.requireDefined(storage);
-        this.logger = commons_1.Contract.requireDefined(logger);
-        this.projectId = ''; // We currently only support coverage for one project.
+        this.internalSessionId = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
+    }
+    /**
+     * If not already done, the bucket to send coverage into for the given application is set up.
+     */
+    async setupBucketAndApplication(bucketSpecifier) {
+        this.commitId = bucketSpecifier.commit;
+        this.appId = bucketSpecifier.appId;
+        await this.storage.setupCoverageBucket(bucketSpecifier);
     }
     /**
      * Put coverage information to the storage for aggregation.
@@ -30,11 +50,18 @@ class Session {
      * @param endLine - The line number within the bundle the range ends.
      */
     putLineCoverage(fileId, startLine, endLine) {
+        commons_1.Contract.requireDefined(this.appId, "The application ID must be set before putting coverage.");
+        commons_1.Contract.requireDefined(this.commitId, "The commit must be set before putting coverage.");
         const lines = [];
         for (let i = startLine; i <= endLine; i++) {
             lines.push(i);
         }
-        this.storage.putCoverage(this.projectId, fileId, lines);
+        this.storage.putCoverage(this.appId, this.commitId, fileId, lines);
+        return lines.length;
+    }
+    /** Returns the internal session ID for debugging purposes. */
+    get sessionId() {
+        return this.internalSessionId;
     }
 }
 exports.Session = Session;