@appland/scanner 1.78.0 → 1.80.0

package/CHANGELOG.md

@@ -1,3 +1,22 @@
+# [@appland/scanner-v1.80.0](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.79.0...@appland/scanner-v1.80.0) (2023-07-19)
+
+
+### Bug Fixes
+
+* Make sure to not process the same appmap multiple times ([da0a0d8](https://github.com/getappmap/appmap-js/commit/da0a0d8826844b4bd92bb671a3ea1b74a5563cb1))
+
+
+### Features
+
+* upgrade @appland/models to 2.6.3 ([6e31f9c](https://github.com/getappmap/appmap-js/commit/6e31f9cc179ac0edfcde2861b937cd104ed4c687))
+
+# [@appland/scanner-v1.79.0](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.78.0...@appland/scanner-v1.79.0) (2023-06-23)
+
+
+### Features
+
+* Modified dates on Finding ([b67d766](https://github.com/getappmap/appmap-js/commit/b67d766722f7d9101f295c7e5fddad5b83e9e032))
+
 # [@appland/scanner-v1.78.0](https://github.com/getappmap/appmap-js/compare/@appland/scanner-v1.77.2...@appland/scanner-v1.78.0) (2023-06-02)
 
 

package/built/cli/scan/watchScan.js

@@ -49,6 +49,8 @@ const telemetry_1 = __importDefault(require("../../telemetry"));
 const events_1 = __importDefault(require("events"));
 const watchScanTelemetry_1 = require("./watchScanTelemetry");
 const isAncestorPath_1 = __importDefault(require("../../util/isAncestorPath"));
+const util_1 = require("util");
+const debug = (0, util_1.debuglog)('scanner:watch');
 function isDir(targetPath) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
@@ -76,6 +78,7 @@ class Watcher {
         // do not remove callbackify, apparently on windows
         // passing plain async function doesn't work (?)
         this.queue = (0, async_1.queue)((0, node_util_1.callbackify)(this.scan.bind(this)), 2);
+        this.processing = new Set();
         watchScanTelemetry_1.WatchScanTelemetry.watch(this.scanEventEmitter, options.appmapDir);
         this.queue.error((error, task) => console.warn(`Problem processing ${task}:\n`, error));
     }
@@ -165,8 +168,9 @@ class Watcher {
         });
     }
     enqueue(mtimePath) {
-        if ([...this.queue].includes(mtimePath))
+        if (this.processing.has(mtimePath))
             return;
+        this.processing.add(mtimePath);
         this.queue.push(mtimePath);
     }
     scan(mtimePath) {
@@ -177,9 +181,11 @@ class Watcher {
             const [appmapStats, reportStats] = yield Promise.all([appmapFile, reportFile].map((f) => (0, promises_1.stat)(f).catch(() => null)));
             if (!appmapStats)
                 return;
+            const cut = (str) => str.substring(str.length - 8);
+            debug('%s: %s, findings: %s, config: %s', appmapFile, cut(appmapStats.mtimeMs.toFixed(3)), reportStats && cut(reportStats.mtimeMs.toFixed(3)), cut(this.config.timestampMs.toFixed(3)));
             if (reportStats &&
-                reportStats.mtimeMs > appmapStats.mtimeMs &&
-                reportStats.mtimeMs > this.config.timestampMs)
+                reportStats.mtimeMs > appmapStats.mtimeMs - 1000 &&
+                reportStats.mtimeMs > this.config.timestampMs - 1000)
                 return; // report is up to date
             const startTime = Date.now();
             const scanner = yield (0, scanner_1.default)(true, this.config, [appmapFile]);
@@ -188,6 +194,7 @@ class Watcher {
             this.scanEventEmitter.emit('scan', { scanResults: rawScanResults, elapsed });
             // Always report the raw data
             yield (0, promises_1.writeFile)(reportFile, (0, formatReport_1.formatReport)(rawScanResults));
+            this.processing.delete(mtimePath);
         });
     }
     reloadConfig() {

package/built/cli/upload.js

@@ -15,20 +15,8 @@ const src_1 = require("@appland/client/dist/src");
 const util_1 = require("../rules/lib/util");
 const create_1 = require("../integration/appland/scannerJob/create");
 const vars_1 = require("../integration/vars");
-const promises_2 = require("fs/promises");
 const path_1 = require("path");
 const pruneAppMap_1 = require("./upload/pruneAppMap");
-function fileExists(file) {
-    return __awaiter(this, void 0, void 0, function* () {
-        try {
-            yield (0, promises_2.stat)(file);
-            return true;
-        }
-        catch (e) {
-            return false;
-        }
-    });
-}
 function create(scanResults, appId, appMapDir, mergeKey, mapsetOptions = {}, retryOptions = {}) {
     return __awaiter(this, void 0, void 0, function* () {
         if ((0, util_1.verbose)())
@@ -47,7 +35,7 @@ function create(scanResults, appId, appMapDir, mergeKey, mapsetOptions = {}, ret
             if ((0, util_1.verbose)())
                 console.log(`Uploading AppMap ${filePath}`);
             const filePaths = [filePath, (0, path_1.join)(appMapDir, filePath)];
-            const filePathsExist = yield Promise.all(filePaths.map(fileExists));
+            const filePathsExist = yield Promise.all(filePaths.map(util_1.fileExists));
             const fullPath = filePaths.find((_, fileIndex) => filePathsExist[fileIndex]);
             if (!fullPath)
                 throw new Error(`File ${filePath} not found`);

package/built/integration/github/commitStatus.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const octokit_1 = require("octokit");
 const vars_1 = require("../vars");
 function postCommitStatus(state, description) {
     (0, vars_1.validateToken)();
@@ -7,9 +8,11 @@ function postCommitStatus(state, description) {
     (0, vars_1.validateOwner)();
     (0, vars_1.validateSha)();
     // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const octokat = require('octokat');
-    const octo = new octokat({ token: (0, vars_1.token)() });
-    return octo.repos((0, vars_1.owner)(), (0, vars_1.repo)()).statuses((0, vars_1.sha)()).create({
+    const octo = new octokit_1.Octokit({ auth: (0, vars_1.token)() });
+    return octo.rest.repos.createCommitStatus({
+        owner: (0, vars_1.owner)(),
+        repo: (0, vars_1.repo)(),
+        sha: (0, vars_1.sha)(),
         state: state,
         context: 'appland/scanner',
         description: description,

package/built/lastGitOrFSModifiedDate.js

@@ -0,0 +1,87 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fileModifiedDate = exports.gitModifiedDate = exports.gitExists = exports.isCached = exports.resetCache = void 0;
+const console_1 = require("console");
+const util_1 = require("./rules/lib/util");
+const child_process_1 = require("child_process");
+const promises_1 = require("fs/promises");
+const FileModifiedDate = new Map();
+let GitExists;
+function resetCache() {
+    FileModifiedDate.clear();
+}
+exports.resetCache = resetCache;
+function isCached(file) {
+    return FileModifiedDate.has(file);
+}
+exports.isCached = isCached;
+function detectGitExists() {
+    return new Promise((resolve) => {
+        (0, child_process_1.exec)('git --version', (err) => {
+            if (err && err.code && err.code > 0)
+                resolve(false);
+            resolve(true);
+        });
+    });
+}
+function gitExists() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (GitExists === undefined) {
+            GitExists = yield detectGitExists();
+        }
+        return GitExists;
+    });
+}
+exports.gitExists = gitExists;
+function gitModifiedDate(file) {
+    return new Promise((resolve) => {
+        (0, child_process_1.exec)(`git log -n 1 --pretty=format:%cI ${file}`, (err, stdout) => {
+            if (err && err.code && err.code > 0)
+                resolve(undefined);
+            if (stdout.trim() === '')
+                resolve(undefined);
+            resolve(new Date(stdout));
+        });
+    });
+}
+exports.gitModifiedDate = gitModifiedDate;
+function fileModifiedDate(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const stats = yield (0, promises_1.stat)(file);
+            return stats.mtime;
+        }
+        catch (e) {
+            (0, console_1.warn)(e);
+        }
+    });
+}
+exports.fileModifiedDate = fileModifiedDate;
+function lastGitOrFSModifiedDate(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let result = FileModifiedDate.get(file);
+        if (result) {
+            if ((0, util_1.verbose)())
+                (0, console_1.debug)(`Using cached modified date for ${file}`);
+            return result.getTime() === 0 ? undefined : result;
+        }
+        if ((0, util_1.verbose)())
+            (0, console_1.debug)(`Computing modified date for ${file}`);
+        if (yield gitExists())
+            result = yield gitModifiedDate(file);
+        if (!result)
+            result = yield fileModifiedDate(file);
+        FileModifiedDate.set(file, result || new Date(0));
+        return result;
+    });
+}
+exports.default = lastGitOrFSModifiedDate;

package/built/ruleChecker.js

@@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const models_1 = require("@appland/models");
 const errors_1 = require("./errors");
 const util_1 = require("./rules/lib/util");
 const rootScope_1 = __importDefault(require("./scope/rootScope"));
@@ -23,6 +24,18 @@ const checkInstance_1 = __importDefault(require("./checkInstance"));
 const eventUtil_1 = require("./eventUtil");
 const hashV1_1 = __importDefault(require("./algorithms/hash/hashV1"));
 const hashV2_1 = __importDefault(require("./algorithms/hash/hashV2"));
+const path_1 = require("path");
+const lastGitOrFSModifiedDate_1 = __importDefault(require("./lastGitOrFSModifiedDate"));
+const console_1 = require("console");
+const assert_1 = __importDefault(require("assert"));
+function locationToFilePath(location) {
+    const [file] = location.split(':');
+    let filePath = file;
+    if ((0, path_1.isAbsolute)(file) && file.startsWith(process.cwd())) {
+        filePath = file.slice(process.cwd().length + 1);
+    }
+    return filePath;
+}
 class RuleChecker {
     constructor(progress) {
         this.progress = progress;
@@ -93,10 +106,44 @@ class RuleChecker {
             if (!checkInstance.filterEvent(event, appMapIndex)) {
                 return;
             }
+            let appmapConfigDir;
+            {
+                let searchDir = (0, path_1.dirname)((0, path_1.resolve)(appMapFileName));
+                while (!appmapConfigDir) {
+                    if (yield (0, util_1.fileExists)((0, path_1.join)(searchDir, 'appmap.yml'))) {
+                        appmapConfigDir = searchDir;
+                    }
+                    else {
+                        if ((0, path_1.dirname)(searchDir) === searchDir)
+                            break;
+                        searchDir = (0, path_1.dirname)(searchDir);
+                    }
+                }
+            }
+            const resolvePath = (path) => __awaiter(this, void 0, void 0, function* () {
+                const candidates = [path];
+                if (appmapConfigDir)
+                    candidates.push((0, path_1.join)(appmapConfigDir, path));
+                for (const candidate of candidates)
+                    if (yield (0, util_1.fileExists)(candidate))
+                        return candidate;
+            });
+            const mostRecentModifiedDate = (filePaths) => __awaiter(this, void 0, void 0, function* () {
+                const dates = new Array();
+                for (const filePath of filePaths) {
+                    const resolvedPath = yield resolvePath(filePath);
+                    if (!resolvedPath)
+                        continue;
+                    const date = yield (0, lastGitOrFSModifiedDate_1.default)(resolvedPath);
+                    if (date)
+                        dates.push(date);
+                }
+                return dates.sort((a, b) => (a && b ? b.getTime() - a.getTime() : 0))[0];
+            });
             const buildFinding = (matchEvent, participatingEvents, message, groupMessage, occurranceCount, 
             // matchEvent will be added to additionalEvents and participatingEvents.values
             // to create the relatedEvents array
-            additionalEvents) => {
+            additionalEvents) => __awaiter(this, void 0, void 0, function* () {
                 const findingEvent = matchEvent || event;
                 // Fixes:
                 // TypeError: Cannot read property 'forEach' of undefined
@@ -112,18 +159,56 @@ class RuleChecker {
                 // findingEvent gets passed here as a relatedEvent, and if you look at HashV1 it
                 // gets added to the hash again. That's how it worked in V1 so it's here for compatibility.
                 additionalEvents || []);
+                let scopeModifiedDate;
+                {
+                    const scopeNavigator = new models_1.EventNavigator(scope);
+                    const scopeFiles = new Set();
+                    const collectScope = (event) => {
+                        if (!event.codeObject.location)
+                            return;
+                        const filePath = locationToFilePath(event.codeObject.location);
+                        if (!filePath)
+                            return;
+                        scopeFiles.add(filePath);
+                    };
+                    collectScope(scope);
+                    for (const descendant of scopeNavigator.descendants()) {
+                        const { event } = descendant;
+                        collectScope(event);
+                    }
+                    const localScopeFiles = [...scopeFiles].filter((filePath) => ((0, assert_1.default)(filePath), !(0, path_1.isAbsolute)(filePath)));
+                    scopeModifiedDate = yield mostRecentModifiedDate(localScopeFiles);
+                }
                 const hashV2 = new hashV2_1.default(checkInstance.ruleId, findingEvent, participatingEvents);
                 const uniqueEvents = new Set();
                 const relatedEvents = [];
-                [findingEvent, ...(additionalEvents || []), ...Object.values(participatingEvents)]
-                    .map(eventUtil_1.cloneEvent)
-                    .forEach((event) => {
+                const relatedEventFiles = new Set();
+                const collectEventFile = (event) => {
+                    if (!event.codeObject.location)
+                        return;
+                    const filePath = locationToFilePath(event.codeObject.location);
+                    if (!filePath)
+                        return;
+                    if ((0, path_1.isAbsolute)(filePath))
+                        return;
+                    relatedEventFiles.add(filePath);
+                };
+                [findingEvent, ...(additionalEvents || []), ...Object.values(participatingEvents)].forEach((event) => {
                     if (uniqueEvents.has(event.id)) {
                         return;
                     }
+                    collectEventFile(event);
+                    for (const ancestor of new models_1.EventNavigator(event).ancestors()) {
+                        collectEventFile(ancestor.event);
+                    }
                     uniqueEvents.add(event.id);
                     relatedEvents.push((0, eventUtil_1.cloneEvent)(event));
                 });
+                const eventsModifiedDate = yield mostRecentModifiedDate([...relatedEventFiles]);
+                if ((0, util_1.verbose)()) {
+                    (0, console_1.warn)(`Scope modified date: ${scopeModifiedDate}`);
+                    (0, console_1.warn)(`Events modified date: ${eventsModifiedDate}`);
+                }
                 return {
                     appMapFile: appMapFileName,
                     checkId: checkInstance.checkId,
@@ -140,8 +225,10 @@ class RuleChecker {
                     relatedEvents: relatedEvents.sort((event) => event.id),
                     impactDomain: checkInstance.checkImpactDomain,
                     participatingEvents: Object.fromEntries(Object.entries(participatingEvents).map(([k, v]) => [k, (0, eventUtil_1.cloneEvent)(v)])),
+                    scopeModifiedDate,
+                    eventsModifiedDate,
                 };
-            };
+            });
             if (this.progress)
                 yield this.progress.matchEvent(event, appMapIndex);
             const matchResult = yield checkInstance.ruleLogic.matcher(event, appMapIndex, checkInstance.filterEvent.bind(checkInstance));
@@ -152,23 +239,23 @@ class RuleChecker {
                 let finding;
                 if (checkInstance.ruleLogic.message) {
                     const message = checkInstance.ruleLogic.message(scope, event);
-                    finding = buildFinding(event, {}, message);
+                    finding = yield buildFinding(event, {}, message);
                 }
                 else {
-                    finding = buildFinding(event, {});
+                    finding = yield buildFinding(event, {});
                 }
                 findings.push(finding);
             }
             else if (typeof matchResult === 'string') {
-                const finding = buildFinding(event, {}, matchResult);
+                const finding = yield buildFinding(event, {}, matchResult);
                 finding.message = matchResult;
                 findings.push(finding);
             }
             else if (matchResult) {
-                matchResult.forEach((mr) => {
-                    const finding = buildFinding(mr.event, mr.participatingEvents || {}, mr.message, mr.groupMessage, mr.occurranceCount, mr.relatedEvents);
+                for (const mr of matchResult) {
+                    const finding = yield buildFinding(mr.event, mr.participatingEvents || {}, mr.message, mr.groupMessage, mr.occurranceCount, mr.relatedEvents);
                     findings.push(finding);
-                });
+                }
             }
             if ((0, util_1.verbose)()) {
                 if (findings.length > numFindings) {

package/built/rules/lib/util.js

@@ -12,11 +12,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.verbose = exports.toRegExpArray = exports.responseContentType = exports.toRegExp = exports.providesAuthentication = exports.pluralize = exports.dasherize = exports.camelize = exports.parseValue = exports.isRoot = exports.ideLink = exports.isTruthy = exports.isFalsey = exports.emptyValue = exports.capitalize = exports.appMapDir = exports.collectAppMapFiles = void 0;
+exports.verbose = exports.toRegExpArray = exports.responseContentType = exports.toRegExp = exports.providesAuthentication = exports.pluralize = exports.dasherize = exports.camelize = exports.parseValue = exports.isRoot = exports.ideLink = exports.isTruthy = exports.isFalsey = exports.fileExists = exports.emptyValue = exports.capitalize = exports.appMapDir = exports.collectAppMapFiles = void 0;
 const path_1 = require("path");
 const util_1 = require("util");
 const glob_1 = require("glob");
 const assert_1 = __importDefault(require("assert"));
+const promises_1 = require("fs/promises");
 function collectAppMapFiles(appmapFile, appmapDir) {
     return __awaiter(this, void 0, void 0, function* () {
         let files = [];
@@ -164,3 +165,15 @@ function pluralize(word, count) {
     return count === 1 ? word : [word, 's'].join('');
 }
 exports.pluralize = pluralize;
+function fileExists(file) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            yield (0, promises_1.stat)(file);
+            return true;
+        }
+        catch (e) {
+            return false;
+        }
+    });
+}
+exports.fileExists = fileExists;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/scanner",
-  "version": "1.78.0",
+  "version": "1.80.0",
   "description": "Analyze AppMaps for code flaws",
   "bin": "built/cli.js",
   "files": [
@@ -20,12 +20,14 @@
     "lint": "eslint src --ext .ts",
     "ci": "yarn lint && yarn build && yarn schema-up-to-date && yarn doc-up-to-date && yarn test",
     "test": "jest --filter=./test/testFilter.js",
+    "jest": "jest --filter=./test/testFilter.js",
     "semantic-release": "semantic-release",
     "watch": "node bin/preBuild.js && tsc -p tsconfig.build.json --watch"
   },
   "author": "AppLand, Inc.",
   "license": "Commons Clause + MIT",
   "devDependencies": {
+    "@appland/appmap-agent-js": "^13.9.0",
     "@semantic-release/changelog": "^6.0.1",
     "@semantic-release/git": "^10.0.1",
     "@types/async": "^3.2.12",
@@ -61,25 +63,25 @@
   },
   "dependencies": {
     "@appland/client": "^1.5.0",
-    "@appland/models": "^2.6.2",
-    "@appland/openapi": "1.5.0",
+    "@appland/models": "^2.6.3",
+    "@appland/openapi": "1.6.0",
     "@appland/sql-parser": "^1.5.0",
     "@types/cli-progress": "^3.9.2",
     "ajv": "^8.8.2",
     "applicationinsights": "^2.1.4",
-    "async": "^3.2.3",
+    "async": "^3.2.4",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
     "chokidar": "^3.5.1",
-    "cli-progress": "^3.11.0",
-    "conf": "^10.0.2",
+    "cli-progress": "^3.12.0",
+    "conf": "10.2.0",
     "crypto-js": "^4.0.0",
     "glob": "7.2.3",
     "inquirer": "^8.1.2",
     "js-yaml": "^4.1.0",
     "lru-cache": "^6.0.0",
     "minimatch": "^5.1.2",
-    "octokat": "^0.10.0",
+    "octokit": "^2.0.19",
     "openapi-diff": "^0.23.5",
     "ora": "~5",
     "pretty-format": "^27.4.6",

package/src/types.d.ts

@@ -116,6 +116,8 @@ interface Finding {
   impactDomain?: ImpactDomain;
   // Map of events by functional role name; for example, logEvent, secret, scope, etc.
   participatingEvents?: Record<string, Event>;
+  scopeModifiedDate?: Date;
+  eventsModifiedDate?: Date;
 }
 
 interface RuleLogic {