@teamscale/coverage-collector 0.0.1-alpha.20 → 0.0.1-beta.22

package/CHANGELOG.md

@@ -0,0 +1,16 @@
+We use [Semantic Versioning](https://semver.org/).
+
+# New Release
+
+# 0.0.1-beta.22
+
+- [feature] Logging of the status text in case of an error when uploading to Teamscale added.  
+
+# 0.0.1-beta.20
+
+- [feature] Docker image of the Coverage Collector now available.
+- [fix] The version of collector and instrumenter are kept in sync from now on.
+
+# 0.0.1-beta.8
+
+- [fix] Reduced the collectors dump interval from 2 minutes to 6 hours. 

package/README.md

@@ -9,7 +9,7 @@ files in the [Teamscale Simple Coverage Format](https://docs.teamscale.com/refer
 The Teamscale JavaScript Profiler consists of this Coverage Collector and the
 [JavaScript Instrumenter](https://www.npmjs.com/package/@teamscale/javascript-instrumenter).
 More details on using them (in combination) can be found 
-on the projects' [Github page](https://github.com/cqse/teamscale-javascript-profiler/).
+in the [Teamscale Documentation](https://docs.teamscale.com/howto/recording-test-coverage-for-javascript/).
 
 The JavaScript Coverage Collector starts a server process that listens for 
 code coverage information from manually or automatically exercised (tested) 
@@ -19,9 +19,10 @@ information back to the original source code.
 ## Building
 
 The Collector is written in TypeScript/JavaScript. For building and running it,
-NodeJs (>= v14) and Yarn (>= v3) are needed as prerequisites.
+NodeJs (>= v14) and Yarn are needed as prerequisites.
 
 ```
+yarn clean
 yarn install
 yarn build
 ```
@@ -31,7 +32,7 @@ yarn build
 There are several options to run the Collector. For example, via `yarn` by running
 
 ```
-yarn serve --port 54678 --dump-to-file=./coverage.simple
+yarn collector --port 54678 --dump-to-file=./coverage.simple
 ```
 
 or via `npx` by running
@@ -48,8 +49,3 @@ for example, by setting a corresponding environment variable.
 export NODE_OPTIONS="$NODE_OPTIONS --max-old-space-size=8192"
 ```
 
-## Publishing
-
-The list of files to publish is defined by the `files` attribute in `package.json`.
-The actual files packed by npm can be listed by running `npx npm-packlist`.
-

package/dist/package.json

@@ -1,27 +1,35 @@
 {
     "name": "@teamscale/coverage-collector",
-    "version": "0.0.1-alpha.18",
+    "version": "0.0.1-beta.21",
     "description": "Collector for JavaScript code coverage information",
-    "main": "dist/main.js",
-    "bin": "dist/main.js",
-    "types": "dist/main.d.ts",
+    "main": "dist/src/main.js",
+    "bin": "dist/src/main.js",
+    "types": "dist/src/main.d.ts",
     "author": "CQSE GmbH",
     "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/cqse/teamscale-javascript-profiler.git"
+    },
     "scripts": {
-        "clean": "npx rimraf dist tsconfig.tsbuildinfo",
+        "clean": "rimraf dist tsconfig.tsbuildinfo",
         "build": "tsc",
-        "serve": "node dist/main.js",
-        "test": "yarn build && jest --forceExit --coverage --silent=true"
+        "collector": "node dist/src/main.js",
+        "test": "yarn build && NODE_OPTIONS='--experimental-vm-modules' jest --forceExit --coverage --silent=true --detectOpenHandles"
     },
     "files": [
         "dist/**/*"
     ],
     "dependencies": {
-        "@cqse/commons": "workspace:*",
+        "@cqse/commons": "^0.0.1-beta.1",
         "argparse": "^2.0.1",
         "async": "^3.2.0",
+        "axios": "^0.24.0",
+        "dotenv": "^14.1.0",
+        "form-data": "^4.0.0",
         "rxjs": "^7.1.0",
         "source-map": "^0.7.3",
+        "tmp": "^0.2.1",
         "typescript-optional": "^2.0.1",
         "winston": "^3.3.3",
         "ws": "^7.4.5"
@@ -34,10 +42,13 @@
         "@types/jest": "^27.0.1",
         "@types/node": "^15.0.1",
         "@types/source-map": "^0.5.7",
+        "@types/tmp": "^0.2.3",
         "@types/winston": "^2.4.4",
         "@types/ws": "^7.4.2",
         "babel-jest": "^27.2.0",
+        "esbuild": "^0.13.4",
         "jest": "^27.2.0",
+        "rimraf": "^3.0.2",
         "ts-jest": "^27.0.5",
         "ts-node": "^10.2.1",
         "typescript": "^4.4.3"

package/dist/src/main.d.ts

@@ -1,7 +1,8 @@
 #!/usr/bin/env node
+import 'dotenv/config';
 /**
  * The main class of the Teamscale JavaScript Collector.
- * Used to start-up the collector for with a given configuration.
+ * Used to start the collector for with a given configuration.
  */
 export declare class Main {
     /**
@@ -23,9 +24,10 @@ export declare class Main {
     /**
      * Start a timer for dumping the data, depending on the configuration.
      *
-     * @param config - The config that determines whether or not to do the timed dump.
+     * @param config - The config that determines whether to do the timed dump or not.
      * @param storage - The storage with the information to dump.
      * @param logger - The logger to use.
      */
     private static maybeStartDumpTimer;
+    private static dumpCoverage;
 }

package/dist/src/main.js

@@ -1,33 +1,104 @@
 #!/usr/bin/env node
-import { version } from '../package.json';
-import { ArgumentParser } from 'argparse';
-import winston from 'winston';
-import { DataStorage } from './storage/DataStorage';
-import { WebSocketCollectingServer } from './receiver/CollectingServer';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Main = void 0;
+const package_json_1 = require("../package.json");
+const argparse_1 = require("argparse");
+const winston_1 = __importDefault(require("winston"));
+const DataStorage_1 = require("./storage/DataStorage");
+const CollectingServer_1 = require("./receiver/CollectingServer");
+require("dotenv/config");
+const fs = __importStar(require("fs"));
+const axios_1 = __importDefault(require("axios"));
+const form_data_1 = __importDefault(require("form-data"));
+const QueryParameters_1 = __importDefault(require("./utils/QueryParameters"));
+const tmp_1 = __importDefault(require("tmp"));
 /**
  * The main class of the Teamscale JavaScript Collector.
- * Used to start-up the collector for with a given configuration.
+ * Used to start the collector for with a given configuration.
  */
-export class Main {
+class Main {
     /**
      * Construct the object for parsing the command line arguments.
      */
     static buildParser() {
-        const parser = new ArgumentParser({
+        var _a;
+        const parser = new argparse_1.ArgumentParser({
             description: 'Collector of the Teamscale JavaScript Profiler. Collects coverage information from a' +
                 '(headless) Web browser that executes code instrumented with our instrumenter.'
         });
-        parser.add_argument('-v', '--version', { action: 'version', version });
+        parser.add_argument('-v', '--version', { action: 'version', version: package_json_1.version });
         parser.add_argument('-p', '--port', { help: 'The port to receive coverage information on.', default: 54678 });
-        parser.add_argument('-f', '--dump-to-file', { help: 'Target file', default: './coverage.simple' });
-        parser.add_argument('-s', '--dump-after-secs', {
-            help: 'Dump the coverage information to the target file every N seconds.',
-            default: 120
+        parser.add_argument('-f', '--dump-to-file', { help: 'Target file to write coverage to.' });
+        parser.add_argument('-l', '--log-to-file', { help: 'Log file', default: 'logs/collector-combined.log' });
+        parser.add_argument('-e', '--log-level', { help: 'Log level', default: 'info' });
+        parser.add_argument('-t', '--dump-after-mins', {
+            help: 'Dump the coverage information to the target file every N minutes.',
+            default: 360
         });
         parser.add_argument('-d', '--debug', {
             help: 'Print received coverage information to the terminal?',
             default: false
         });
+        // Parameters for the upload to Teamscale
+        parser.add_argument('-u', '--teamscale-server-url', {
+            help: 'Upload the coverage to the given Teamscale server URL, for example, https://teamscale.dev.example.com:8080/production.',
+            default: process.env.TEAMSCALE_SERVER_URL
+        });
+        parser.add_argument('--teamscale-access-token', {
+            help: 'The API key to use for uploading to Teamscale.',
+            default: process.env.TEAMSCALE_ACCESS_TOKEN
+        });
+        parser.add_argument('--teamscale-project', {
+            help: 'The project ID to upload coverage to.',
+            default: process.env.TEAMSCALE_PROJECT
+        });
+        parser.add_argument('--teamscale-user', {
+            help: 'The user for uploading coverage to Teamscale.',
+            default: process.env.TEAMSCALE_USER
+        });
+        parser.add_argument('--teamscale-partition', {
+            help: 'The partition to upload coverage to.',
+            default: process.env.TEAMSCALE_PARTITION
+        });
+        parser.add_argument('--teamscale-revision', {
+            help: 'The revision (commit hash, version id) to upload coverage for.',
+            default: process.env.TEAMSCALE_REVISION
+        });
+        parser.add_argument('--teamscale-commit', {
+            help: 'The branch and timestamp to upload coverage for, separated by colon.',
+            default: process.env.TEAMSCALE_COMMIT
+        });
+        parser.add_argument('--teamscale-repository', {
+            help: 'The repository to upload coverage for. Optional: Only needed when uploading via revision to a project that has more than one connector.',
+            default: process.env.TEAMSCALE_REPOSITORY
+        });
+        parser.add_argument('--teamscale-message', {
+            help: 'The commit message shown within Teamscale for the coverage upload. Default is "JavaScript coverage upload".',
+            default: (_a = process.env.TEAMSCALE_MESSAGE) !== null && _a !== void 0 ? _a : 'JavaScript coverage upload'
+        });
         return parser;
     }
     /**
@@ -40,15 +111,15 @@ export class Main {
     /**
      * Construct the logger.
      */
-    static buildLogger() {
-        return winston.createLogger({
-            level: 'info',
-            format: winston.format.json(),
+    static buildLogger(config) {
+        return winston_1.default.createLogger({
+            level: config.log_level,
+            format: winston_1.default.format.json(),
             defaultMeta: {},
             transports: [
-                new winston.transports.File({ filename: 'logs/collector-error.log', level: 'error' }),
-                new winston.transports.File({ filename: 'logs/collector-combined.log' }),
-                new winston.transports.Console({ format: winston.format.simple(), level: 'info' })
+                new winston_1.default.transports.File({ filename: 'logs/collector-error.log', level: 'error' }),
+                new winston_1.default.transports.File({ filename: config.log_to_file.trim() }),
+                new winston_1.default.transports.Console({ format: winston_1.default.format.simple(), level: config.log_level })
             ]
         });
     }
@@ -56,20 +127,29 @@ export class Main {
      * Entry point of the Teamscale JavaScript Profiler.
      */
     static run() {
-        const logger = this.buildLogger();
-        logger.info(`Starting collector in working directory "${process.cwd()}".`);
         // Parse the command line arguments
         const config = this.parseArguments();
+        // Build the logger
+        const logger = this.buildLogger(config);
+        logger.info(`Starting collector in working directory "${process.cwd()}".`);
+        logger.info(`Logging "${config.log_level}" to "${config.log_to_file}".`);
+        // Check the command line arguments
+        if (!config.dump_to_file && !config.teamscale_server_url) {
+            logger.error('The Collector must be configured to either dump to a file or upload to Teamscale.');
+            process.exit(1);
+        }
         // Prepare the storage and the server
-        const storage = new DataStorage(logger);
-        const server = new WebSocketCollectingServer(config.port, storage, logger);
+        const storage = new DataStorage_1.DataStorage(logger);
+        const server = new CollectingServer_1.WebSocketCollectingServer(config.port, storage, logger);
         // Start the server socket.
         // ATTENTION: The server is executed asynchronously
         server.start();
         // Optionally, start a timer that dumps the coverage after a N seconds
         this.maybeStartDumpTimer(config, storage, logger);
         // Say bye bye on CTRL+C and exit the process
-        process.on('SIGINT', () => {
+        process.on('SIGINT', async () => {
+            // ... and do a final dump before.
+            await this.dumpCoverage(config, storage, logger).then();
             logger.info('Bye bye.');
             process.exit();
         });
@@ -77,31 +157,82 @@ export class Main {
     /**
      * Start a timer for dumping the data, depending on the configuration.
      *
-     * @param config - The config that determines whether or not to do the timed dump.
+     * @param config - The config that determines whether to do the timed dump or not.
      * @param storage - The storage with the information to dump.
      * @param logger - The logger to use.
      */
     static maybeStartDumpTimer(config, storage, logger) {
-        if (config.dump_after_secs > 0) {
+        if (config.dump_after_mins > 0) {
+            logger.info(`Will dump coverage information every ${config.dump_after_mins} minute(s).`);
             const timer = setInterval(() => {
-                try {
-                    const lines = storage.dumpToSimpleCoverageFile(config.dump_to_file);
-                    logger.info(`Conducted periodic coverage dump with ${lines} lines to ${config.dump_to_file}.`);
-                }
-                catch (e) {
-                    logger.error('Timed coverage dump failed.', e);
-                }
-            }, config.dump_after_secs * 1000);
+                this.dumpCoverage(config, storage, logger).then();
+            }, config.dump_after_mins * 1000 * 60);
             process.on('SIGINT', () => {
                 // Stop the timed file dump
                 if (timer) {
                     clearInterval(timer);
                 }
-                // ... and do a final dump
-                logger.info('\nCaught interrupt signal. Writing latest coverage.');
-                storage.dumpToSimpleCoverageFile(config.dump_to_file);
             });
         }
     }
+    static async dumpCoverage(config, storage, logger) {
+        var _a;
+        try {
+            const deleteCoverageFileAfterUpload = !config.dump_to_file;
+            const coverageFile = (_a = config.dump_to_file) !== null && _a !== void 0 ? _a : tmp_1.default.tmpNameSync();
+            try {
+                // 1. Write coverage to a file
+                const lines = storage.dumpToSimpleCoverageFile(coverageFile);
+                logger.info(`Dumped ${lines} lines of coverage to ${coverageFile}.`);
+                // 2. Upload to Teamscale if configured
+                if (!config.teamscale_server_url) {
+                    logger.info('Upload to Teamscale not configured.');
+                }
+                else if (lines === 0) {
+                    // Nothing to upload
+                }
+                else if (config.teamscale_access_token && config.teamscale_user) {
+                    logger.info('Preparing upload to Teamscale');
+                    const form = new form_data_1.default();
+                    form.append('report', fs.createReadStream(coverageFile), 'coverage.simple');
+                    const parameters = new QueryParameters_1.default();
+                    parameters.addIfDefined('format', 'SIMPLE');
+                    parameters.addIfDefined('message', config.teamscale_message);
+                    parameters.addIfDefined('repository', config.teamscale_repository);
+                    parameters.addIfDefined('t', config.teamscale_commit);
+                    parameters.addIfDefined('revision', config.teamscale_revision);
+                    parameters.addIfDefined('partition', config.teamscale_partition);
+                    const response = await axios_1.default.post(`${config.teamscale_server_url.replace(/\/$/, '')}/api/projects/${config.teamscale_project}/external-analysis/session/auto-create/report?${parameters.toString()}`, form, {
+                        auth: {
+                            username: config.teamscale_user,
+                            password: config.teamscale_access_token
+                        },
+                        headers: {
+                            Accept: '*/*',
+                            'Content-Type': `multipart/form-data; boundary=${form.getBoundary()}`
+                        }
+                    });
+                    if (response.status >= 400) {
+                        logger.error(`Upload failed with code ${response.status}: ${response.statusText}`);
+                    }
+                    else {
+                        logger.info(`Upload with status code ${response.status} finished.`);
+                    }
+                }
+                else {
+                    logger.error('Cannot upload to Teamscale: API key and user name must be configured!');
+                }
+            }
+            finally {
+                if (deleteCoverageFileAfterUpload) {
+                    fs.unlinkSync(coverageFile);
+                }
+            }
+        }
+        catch (e) {
+            logger.error('Coverage dump failed.', e);
+        }
+    }
 }
+exports.Main = Main;
 Main.run();

package/dist/src/receiver/CollectingServer.js

@@ -1,17 +1,39 @@
-import * as WebSocket from 'ws';
-import { Contract } from '@cqse/commons';
-import { Session } from './Session';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebSocketCollectingServer = exports.ProtocolMessageTypes = void 0;
+const WebSocket = __importStar(require("ws"));
+const commons_1 = require("@cqse/commons");
+const Session_1 = require("./Session");
 /**
  * Various constants that are used to exchange data between
  * the instrumented application and the coverage collector.
  */
-export var ProtocolMessageTypes;
+var ProtocolMessageTypes;
 (function (ProtocolMessageTypes) {
     /** A message that provides a source map */
     ProtocolMessageTypes["TYPE_SOURCEMAP"] = "s";
     /** A message that provides coverage information */
     ProtocolMessageTypes["TYPE_COVERAGE"] = "c";
-})(ProtocolMessageTypes || (ProtocolMessageTypes = {}));
+})(ProtocolMessageTypes = exports.ProtocolMessageTypes || (exports.ProtocolMessageTypes = {}));
 /**
  * Separates the instrumentation subject from the coverage information.
  */
@@ -20,7 +42,7 @@ const INSTRUMENTATION_SUBJECT_SEPARATOR = ':';
  * A WebSocket based implementation of a coverage receiver.
  * Receives coverage from instrumented JavaScript code.
  */
-export class WebSocketCollectingServer {
+class WebSocketCollectingServer {
     /**
      * Constructor.
      *
@@ -29,9 +51,9 @@ export class WebSocketCollectingServer {
      * @param logger - The logger to use.
      */
     constructor(port, storage, logger) {
-        Contract.require(port > 0 && port < 65536, 'Port must be valid (range).');
-        this.storage = Contract.requireDefined(storage);
-        this.logger = Contract.requireDefined(logger);
+        commons_1.Contract.require(port > 0 && port < 65536, 'Port must be valid (range).');
+        this.storage = commons_1.Contract.requireDefined(storage);
+        this.logger = commons_1.Contract.requireDefined(logger);
         this.server = new WebSocket.Server({ port: port });
     }
     /**
@@ -41,7 +63,7 @@ export class WebSocketCollectingServer {
         this.logger.info(`Starting server on port ${this.server.options.port}.`);
         // Handle new connections from clients
         this.server.on('connection', (webSocket, req) => {
-            let session = new Session(req.socket, this.storage, this.logger);
+            let session = new Session_1.Session(req.socket, this.storage, this.logger);
             this.logger.debug(`Connection from: ${req.socket.remoteAddress}`);
             // Handle disconnecting clients
             webSocket.on('close', code => {
@@ -94,6 +116,7 @@ export class WebSocketCollectingServer {
         const fileIdSeparatorPosition = body.indexOf(INSTRUMENTATION_SUBJECT_SEPARATOR);
         if (fileIdSeparatorPosition > -1) {
             const fileId = body.substring(0, fileIdSeparatorPosition).trim();
+            this.logger.debug(`Received source map information for ${fileId}`);
             const sourcemap = body.substring(fileIdSeparatorPosition + 1);
             session.putSourcemap(fileId, sourcemap);
         }
@@ -118,3 +141,4 @@ export class WebSocketCollectingServer {
         }
     }
 }
+exports.WebSocketCollectingServer = WebSocketCollectingServer;

package/dist/src/receiver/Session.js

@@ -1,11 +1,33 @@
-import * as sourceMap from 'source-map';
-import { Contract } from '@cqse/commons';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Session = void 0;
+const sourceMap = __importStar(require("source-map"));
+const commons_1 = require("@cqse/commons");
 /**
  * The session maintains the relevant information for a client.
  * One session is created for each client.
  * The mapping based on sourcemaps is conducted here.
  */
-export class Session {
+class Session {
     /**
      * Constructor
      *
@@ -14,9 +36,9 @@ export class Session {
      * @param logger - The logger to use.
      */
     constructor(socket, storage, logger) {
-        this.socket = Contract.requireDefined(socket);
-        this.storage = Contract.requireDefined(storage);
-        this.logger = Contract.requireDefined(logger);
+        this.socket = commons_1.Contract.requireDefined(socket);
+        this.storage = commons_1.Contract.requireDefined(storage);
+        this.logger = commons_1.Contract.requireDefined(logger);
         this.sourceMaps = new Map();
         this.projectId = ''; // We currently only support coverage for one project.
     }
@@ -84,3 +106,4 @@ export class Session {
         }
     }
 }
+exports.Session = Session;

package/dist/src/storage/DataStorage.d.ts

@@ -94,6 +94,10 @@ export declare class DataStorage implements IDataStorage {
      * Logger to use.
      */
     private readonly logger;
+    /**
+     * Times unmapped coverage received.
+     */
+    private timesUnmappedCoverage;
     /**
      * Constructs the data storage.
      *