@appland/scanner 1.72.0 → 1.73.1

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+# [@appland/scanner-v1.73.1](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.73.0...@appland/scanner-v1.73.1) (2022-11-29)
+
+
+### Bug Fixes
+
+* Continue watching correctly after skipping files ([b5d7c2d](https://github.com/getappmap/appmap-js/commit/b5d7c2df6da78ae20b6547e0ddd23f41f20cb428))
+
+# [@appland/scanner-v1.73.0](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.72.0...@appland/scanner-v1.73.0) (2022-11-28)
+
+
+### Bug Fixes
+
+* Address PR comments ([788dab4](https://github.com/getappmap/appmap-js/commit/788dab46cea2933dc0c5965f6307f5c0ef4d2873))
+* sendTelemetry uses ScanResults.aggregate ([14e5bbb](https://github.com/getappmap/appmap-js/commit/14e5bbb074e4153878875a9376cbf10330baa1b4))
+
+
+### Features
+
+* Send scan:started at beginning of analysis and send data in ([76d2141](https://github.com/getappmap/appmap-js/commit/76d2141fdadabeb21f15af86f879e76ff0389237))
+
 # [@appland/scanner-v1.72.0](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.71.7...@appland/scanner-v1.72.0) (2022-11-01)
 
 

package/built/cli/scan/singleScan.js

@@ -20,11 +20,18 @@ const findingsReport_1 = __importDefault(require("../../report/findingsReport"))
 const summaryReport_1 = __importDefault(require("../../report/summaryReport"));
 const formatReport_1 = require("./formatReport");
 const telemetry_1 = __importDefault(require("../../telemetry"));
+const scanResults_1 = require("../../report/scanResults");
 const util_1 = require("../../rules/lib/util");
 const validateFile_1 = __importDefault(require("../validateFile"));
 function singleScan(options) {
     return __awaiter(this, void 0, void 0, function* () {
         const { appmapFile, appmapDir, configuration, reportAllFindings, appId, ide, reportFile } = options;
+        telemetry_1.default.sendEvent({
+            name: 'scan:started',
+            properties: {
+                ide,
+            },
+        });
         const skipErrors = appmapDir !== undefined;
         const files = yield (0, util_1.collectAppMapFiles)(appmapFile, appmapDir);
         yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () { return (0, validateFile_1.default)('file', file); })));
@@ -52,22 +59,12 @@ function singleScan(options) {
         const elapsed = Date.now() - startTime;
         const numChecks = scanResults.checks.length * scanResults.summary.numAppMaps;
         console.log(`Performed ${numChecks} checks in ${elapsed}ms (${Math.floor(numChecks / (elapsed / 1000.0))} checks/sec)`);
-        sendTelemetry(scanResults, elapsed);
-    });
-}
-exports.default = singleScan;
-function sendTelemetry(scanResults, msElapsed) {
-    const rules = [...new Set(scanResults.checks.map(({ id }) => id))];
-    telemetry_1.default.sendEvent({
-        name: 'scan:completed',
-        properties: {
-            rules: rules.join(', '),
-        },
-        metrics: {
-            duration: msElapsed / 1000,
-            numRules: rules.length,
+        (0, scanResults_1.sendScanResultsTelemetry)({
+            ruleIds: scanResults.summary.rules,
             numAppMaps: scanResults.summary.numAppMaps,
-            numFindings: scanResults.findings.length,
-        },
+            numFindings: scanResults.summary.numFindings,
+            elapsedMs: elapsed,
+        });
     });
 }
+exports.default = singleScan;

package/built/cli/scan/watchScan.js

@@ -38,12 +38,15 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Watcher = void 0;
 const promises_1 = require("fs/promises");
 const chokidar = __importStar(require("chokidar"));
-const formatReport_1 = require("./formatReport");
-const scanner_1 = __importDefault(require("./scanner"));
-const configurationProvider_1 = require("../../configuration/configurationProvider");
 const assert_1 = __importDefault(require("assert"));
 const path_1 = __importDefault(require("path"));
 const async_1 = require("async");
+const node_util_1 = require("node:util");
+const formatReport_1 = require("./formatReport");
+const scanner_1 = __importDefault(require("./scanner"));
+const configurationProvider_1 = require("../../configuration/configurationProvider");
+const telemetry_1 = __importDefault(require("../../telemetry"));
+const events_1 = __importDefault(require("events"));
 function isDir(targetPath) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
@@ -70,11 +73,17 @@ function isAncestorPath(ancestor, descendant) {
 class Watcher {
     constructor(options) {
         this.options = options;
-        this.queue = (0, async_1.queue)(this.scan.bind(this), 2);
+        this.scanEventEmitter = new events_1.default();
+        // do not remove callbackify, apparently on windows
+        // passing plain async function doesn't work (?)
+        this.queue = (0, async_1.queue)((0, node_util_1.callbackify)(this.scan.bind(this)), 2);
     }
     watch() {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.reloadConfig();
+            telemetry_1.default.sendEvent({
+                name: 'scan:started',
+            });
             this.configWatcher = chokidar.watch(this.options.configFile, {
                 ignoreInitial: true,
             });
@@ -128,7 +137,7 @@ class Watcher {
             return;
         this.queue.push(mtimePath);
     }
-    scan(mtimePath, callback) {
+    scan(mtimePath) {
         return __awaiter(this, void 0, void 0, function* () {
             (0, assert_1.default)(this.config, `config should always be loaded before appmapWatcher triggers a scan`);
             const appmapFile = [path_1.default.dirname(mtimePath), 'appmap.json'].join('.');
@@ -140,14 +149,13 @@ class Watcher {
                 reportStats.mtimeMs > appmapStats.mtimeMs &&
                 reportStats.mtimeMs > this.config.timestampMs)
                 return; // report is up to date
+            const startTime = Date.now();
             const scanner = yield (0, scanner_1.default)(true, this.config, [appmapFile]);
             const rawScanResults = yield scanner.scan();
+            const elapsed = Date.now() - startTime;
+            this.scanEventEmitter.emit('scan', { scanResults: rawScanResults, elapsed });
             // Always report the raw data
             yield (0, promises_1.writeFile)(reportFile, (0, formatReport_1.formatReport)(rawScanResults));
-            // tell the queue that the current task is complete
-            if (callback) {
-                yield callback();
-            }
         });
     }
     reloadConfig() {

package/built/cli/scan/watchScanTelemetry.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WatchScanTelemetry = void 0;
+const eventAggregator_1 = __importDefault(require("../../lib/eventAggregator"));
+const scanResults_1 = require("../../report/scanResults");
+class WatchScanTelemetry {
+    constructor(scanEvents) {
+        new eventAggregator_1.default((events) => {
+            const scanEvents = events.map((e) => e.arg);
+            this.sendTelemetry(scanEvents);
+        }).attach(scanEvents, 'scan');
+    }
+    sendTelemetry(scanEvents) {
+        const ruleIds = new Set();
+        let elapsed = 0;
+        const telemetryScanResults = new scanResults_1.ScanResults();
+        for (const scanEvent of scanEvents) {
+            telemetryScanResults.aggregate(scanEvent.scanResults);
+            elapsed += scanEvent.elapsed;
+        }
+        telemetryScanResults.summary.rules.forEach((rule) => ruleIds.add(rule));
+        (0, scanResults_1.sendScanResultsTelemetry)({
+            ruleIds: [...ruleIds],
+            numAppMaps: telemetryScanResults.summary.numAppMaps,
+            numFindings: telemetryScanResults.summary.numFindings,
+            elapsedMs: elapsed,
+        });
+    }
+}
+exports.WatchScanTelemetry = WatchScanTelemetry;

package/built/lib/eventAggregator.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MaxMSBetween = void 0;
+exports.MaxMSBetween = 10 * 1000;
+// TODO: Unify with the code in packages/cli - find a way to make a common import.
+class EventAggregator {
+    constructor(callback, maxMsBetween = exports.MaxMSBetween) {
+        this.callback = callback;
+        this.maxMsBetween = maxMsBetween;
+        this.pending = [];
+        process.on('exit', () => {
+            if (this.timeout) {
+                clearTimeout(this.timeout);
+                this.emitPending();
+            }
+        });
+    }
+    push(emitter, event, arg) {
+        this.pending.push({ emitter, event, arg });
+        this.refresh();
+    }
+    refresh() {
+        if (this.timeout)
+            clearTimeout(this.timeout);
+        this.timeout = setTimeout(this.emitPending.bind(this), this.maxMsBetween);
+    }
+    emitPending() {
+        this.callback(this.pending);
+        this.timeout = undefined;
+        this.pending = [];
+    }
+    attach(emitter, event) {
+        emitter.on(event, (...args) => this.push(emitter, event, args[0]));
+    }
+}
+exports.default = EventAggregator;

package/built/report/scanResults.js

@@ -1,6 +1,10 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScanResults = void 0;
+exports.sendScanResultsTelemetry = exports.ScanResults = void 0;
+const telemetry_1 = __importDefault(require("../telemetry"));
 class DistinctItems {
     constructor() {
         this.members = {};
@@ -55,7 +59,7 @@ function collectMetadata(metadata) {
  * It's used for printing a user-friendly summary report, it's not used for machine-readable program output.
  */
 class ScanResults {
-    constructor(configuration, appMapMetadata, findings, checks) {
+    constructor(configuration = { checks: [] }, appMapMetadata = {}, findings = [], checks = []) {
         this.configuration = configuration;
         this.appMapMetadata = appMapMetadata;
         this.findings = findings;
@@ -72,5 +76,31 @@ class ScanResults {
     withFindings(findings) {
         return new ScanResults(this.configuration, this.appMapMetadata, findings, this.checks);
     }
+    aggregate(sourceScanResults) {
+        this.summary.numAppMaps += sourceScanResults.summary.numAppMaps;
+        this.summary.numChecks += sourceScanResults.summary.numChecks;
+        this.summary.rules = [...new Set(this.summary.rules.concat(sourceScanResults.summary.rules))];
+        this.summary.ruleLabels = [
+            ...new Set(this.summary.ruleLabels.concat(sourceScanResults.summary.ruleLabels)),
+        ];
+        this.summary.numFindings += sourceScanResults.summary.numFindings;
+        // we don't need sourceScanResults.summary.appMetadata
+        // appMapMetadata.Git may also contain secrets we don't want to transmit.
+    }
 }
 exports.ScanResults = ScanResults;
+function sendScanResultsTelemetry(telemetry) {
+    telemetry_1.default.sendEvent({
+        name: 'scan:completed',
+        properties: {
+            rules: telemetry.ruleIds.sort().join(', '),
+        },
+        metrics: {
+            duration: telemetry.elapsedMs / 1000,
+            numRules: telemetry.ruleIds.length,
+            numAppMaps: telemetry.numAppMaps,
+            numFindings: telemetry.numFindings,
+        },
+    });
+}
+exports.sendScanResultsTelemetry = sendScanResultsTelemetry;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/scanner",
-  "version": "1.72.0",
+  "version": "1.73.1",
   "description": "Analyze AppMaps for code flaws",
   "bin": "built/cli.js",
   "files": [
@@ -61,7 +61,7 @@
   "dependencies": {
     "@appland/client": "^1.3.0",
     "@appland/models": "^1.18.1",
-    "@appland/openapi": "1.1.0",
+    "@appland/openapi": "1.2.0",
     "@appland/sql-parser": "^1.5.0",
     "@types/cli-progress": "^3.9.2",
     "ajv": "^8.8.2",