@appland/scanner 1.62.1 → 1.64.0

package/CHANGELOG.md

@@ -1,3 +1,26 @@
+# [@appland/scanner-v1.64.0](https://github.com/applandinc/appmap-js/compare/@appland/scanner-v1.63.0...@appland/scanner-v1.64.0) (2022-08-04)
+
+
+### Features
+
+* Command scope falls back on root events ([3823a1f](https://github.com/applandinc/appmap-js/commit/3823a1f686212db49b87f2995baa2103a4e007d1))
+
+# [@appland/scanner-v1.63.0](https://github.com/applandinc/appmap-js/compare/@appland/scanner-v1.62.2...@appland/scanner-v1.63.0) (2022-07-28)
+
+
+### Features
+
+* Include a partial stack in the finding hash ([7e82f8a](https://github.com/applandinc/appmap-js/commit/7e82f8a0b13a1d0927aad73be4ee126d2d4695dc))
+* Populate hash_v2 on each finding ([04470b7](https://github.com/applandinc/appmap-js/commit/04470b7f11e764d79a22eb297d0e6882f6f89a3f))
+* Summarize local report using hash_v2 ([ffbde39](https://github.com/applandinc/appmap-js/commit/ffbde393c17f1f1572eb7653bad796d90662b943))
+
+# [@appland/scanner-v1.62.2](https://github.com/applandinc/appmap-js/compare/@appland/scanner-v1.62.1...@appland/scanner-v1.62.2) (2022-07-25)
+
+
+### Bug Fixes
+
+* Return proper result for job-not-cancelled ([f7ee5da](https://github.com/applandinc/appmap-js/commit/f7ee5da073849881c3c553f08fc2dd82bb8c7965))
+
 # [@appland/scanner-v1.62.1](https://github.com/applandinc/appmap-js/compare/@appland/scanner-v1.62.0...@appland/scanner-v1.62.1) (2022-07-13)
 
 

package/built/algorithms/hash/hashV1.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto_1 = require("crypto");
+class HashV1 {
+    constructor(ruleId, findingEvent, relatedEvents) {
+        this.hash = (0, crypto_1.createHash)('sha256');
+        this.hash.update(findingEvent.hash);
+        this.hash.update(ruleId);
+        // Admittedly odd, this implementation matches the original hash algorithm.
+        const uniqueEvents = new Set();
+        const hashEvents = [];
+        relatedEvents.unshift(findingEvent);
+        relatedEvents.forEach((event) => {
+            if (uniqueEvents.has(event.id))
+                return;
+            uniqueEvents.add(event.id);
+            hashEvents.push(event);
+        });
+        // This part where the hashes go into a Set, and there is some kind of ordering as a side-
+        // effect, is particularly weird.
+        new Set(hashEvents.map((e) => e.hash)).forEach((eventHash) => {
+            this.hash.update(eventHash);
+        });
+    }
+    digest() {
+        return this.hash.digest('hex');
+    }
+}
+exports.default = HashV1;

package/built/algorithms/hash/hashV2.js

@@ -0,0 +1,107 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureStack = void 0;
+const crypto_1 = require("crypto");
+const isCommand_1 = __importDefault(require("../../rules/lib/isCommand"));
+const util_1 = require("../../rules/lib/util");
+function hashEvent(entries, prefix, event) {
+    Object.keys(event.stableProperties)
+        .sort()
+        .forEach((key) => entries.push([[prefix, key].join('.'), event.stableProperties[key].toString()].join('=')));
+}
+const STACK_DEPTH = 3;
+/**
+ * Captures stack entries from distinct packages. Ancestors of the event are traversed up to the
+ * command or root. Then, starting from the command or root, subsequent events which come from the
+ * same package as their preceding event are removed. Then the last N entries remaining in the
+ * stack are collected.
+ *
+ * @param event leaf event
+ * @param participatingEvents output collector
+ * @param depth number of events to include in the output
+ */
+function captureStack(event, depth = STACK_DEPTH) {
+    let ancestor = event.parent;
+    const stack = [];
+    while (ancestor) {
+        stack.push(ancestor);
+        ancestor = (0, isCommand_1.default)(ancestor) ? undefined : ancestor.parent;
+    }
+    const packageOf = (event) => {
+        if (!event)
+            return;
+        if (event.codeObject.type !== 'function')
+            return;
+        return event.codeObject.packageOf;
+    };
+    return stack
+        .filter((item, index) => item.codeObject.type !== 'function' || packageOf(stack[index + 1]) !== packageOf(item))
+        .slice(0, depth);
+}
+exports.captureStack = captureStack;
+/**
+ * Builds a hash (digest) of a finding. The digest is constructed by first building a canonical
+ * string of the finding, of the form:
+ *
+ * ```
+ * [
+ *   algorithmVersion=2
+ *   rule=<rule-id>
+ *   findingEvent.<property1>=value1
+ *   ...
+ *   findingEvent.<propertyN>=valueN
+ *   participatingEvent.<eventName1>=value1
+ *   ...
+ *   participatingEvent.<eventName1>=valueN
+ *   ...
+ *   participatingEvent.<eventNameN>=value1
+ *   ...
+ *   participatingEvent.<eventNameN>=valueN
+ *   stack[1]=value1
+ *   ...
+ *   stack[1]=valueN
+ *   ...
+ *   stack[3]=value1
+ *   ...
+ *   stack[3]=valueN
+ * ]
+ * ```
+ *
+ * Participating events are sorted by the event name. Properties of each event are sorted by
+ * the property name. Event properties are provided by `Event#stableProperties`.
+ *
+ * The partial stack included in the finding hash removes subsequent function calls from the
+ * same package.
+ */
+class HashV2 {
+    constructor(ruleId, findingEvent, participatingEvents) {
+        this.hashEntries = [];
+        this.hash = (0, crypto_1.createHash)('sha256');
+        const hashEntries = [
+            ['algorithmVersion', '2'],
+            ['rule', ruleId],
+        ].map((e) => e.join('='));
+        this.hashEntries = hashEntries;
+        hashEvent(hashEntries, 'findingEvent', findingEvent);
+        Object.keys(participatingEvents)
+            .sort()
+            .forEach((key) => {
+            const event = participatingEvents[key];
+            hashEvent(hashEntries, `participatingEvent.${key}`, event);
+        });
+        captureStack(findingEvent).forEach((event, index) => hashEvent(hashEntries, `stack[${index + 1}]`, event));
+        if ((0, util_1.verbose)())
+            console.log(hashEntries);
+        hashEntries.forEach((e) => this.hash.update(e));
+    }
+    get canonicalString() {
+        return this.hashEntries.join('\n');
+    }
+    digest() {
+        return this.hash.digest('hex');
+    }
+}
+exports.default = HashV2;

package/built/check.js

@@ -9,7 +9,7 @@ class Check {
         }
         this.id = rule.id;
         this.options = options || makeOptions();
-        this.scope = rule.scope || 'root';
+        this.scope = rule.scope || 'command';
         this.includeScope = [];
         this.excludeScope = [];
         this.includeEvent = [];

package/built/report/summaryReport.js

@@ -10,8 +10,8 @@ function summarizeFindings(findings) {
         let findingSummary = memo[finding.ruleId];
         if (findingSummary) {
             findingSummary.findingTotal += 1;
-            if (!findingSummary.findingHashes.has(finding.hash)) {
-                findingSummary.findingHashes.add(finding.hash);
+            if (!findingSummary.findingHashes.has(finding.hash_v2)) {
+                findingSummary.findingHashes.add(finding.hash_v2);
                 findingSummary.messages.push(finding.message);
             }
         }
@@ -20,7 +20,7 @@ function summarizeFindings(findings) {
                 ruleId: finding.ruleId,
                 ruleTitle: finding.ruleTitle,
                 findingTotal: 1,
-                findingHashes: new Set([finding.hash]),
+                findingHashes: new Set([finding.hash_v2]),
                 messages: [finding.message],
             };
             memo[finding.ruleId] = findingSummary;
@@ -31,7 +31,7 @@ function summarizeFindings(findings) {
     return Object.values(result);
 }
 function default_1(summary, colorize) {
-    const matchedStr = `${summary.summary.numFindings} ${(0, util_1.pluralize)('finding', summary.summary.numFindings)} (${new Set(summary.findings.map((finding) => finding.hash)).size} unique)`;
+    const matchedStr = `${summary.summary.numFindings} ${(0, util_1.pluralize)('finding', summary.summary.numFindings)} (${new Set(summary.findings.map((finding) => finding.hash_v2)).size} unique)`;
     const colouredMatchedStr = colorize ? chalk_1.default.stderr.magenta(matchedStr) : matchedStr;
     console.log();
     console.log(colouredMatchedStr);

package/built/ruleChecker.js

@@ -20,8 +20,9 @@ const httpClientRequestScope_1 = __importDefault(require("./scope/httpClientRequ
 const commandScope_1 = __importDefault(require("./scope/commandScope"));
 const sqlTransactionScope_1 = __importDefault(require("./scope/sqlTransactionScope"));
 const checkInstance_1 = __importDefault(require("./checkInstance"));
-const crypto_1 = require("crypto");
 const eventUtil_1 = require("./eventUtil");
+const hashV1_1 = __importDefault(require("./algorithms/hash/hashV1"));
+const hashV2_1 = __importDefault(require("./algorithms/hash/hashV2"));
 class RuleChecker {
     constructor() {
         this.scopes = {
@@ -34,14 +35,7 @@ class RuleChecker {
     }
     check(appMapFile, appMapIndex, check, findings) {
         return __awaiter(this, void 0, void 0, function* () {
-            const numScopesChecked = yield this.checkScope(appMapFile, appMapIndex, check, check.scope, findings);
-            if (numScopesChecked === 0 && check.scope === 'command') {
-                yield this.checkScope(appMapFile, appMapIndex, check, 'root', findings);
-            }
-        });
-    }
-    checkScope(appMapFile, appMapIndex, check, scope, findings) {
-        return __awaiter(this, void 0, void 0, function* () {
+            const scope = check.scope;
             if ((0, util_1.verbose)()) {
                 console.warn(`Checking AppMap ${appMapIndex.appMap.name} with scope ${scope}`);
             }
@@ -55,9 +49,7 @@ class RuleChecker {
                     yield events[i];
                 }
             };
-            let numScopes = 0;
             for (const scope of scopeIterator.scopes(callEvents())) {
-                numScopes += 1;
                 if ((0, util_1.verbose)()) {
                     console.warn(`Scope ${scope.scope}`);
                 }
@@ -74,7 +66,6 @@ class RuleChecker {
                     yield this.checkEvent(scope.scope, scope.scope, appMapFile, appMapIndex, checkInstance, findings);
                 }
             }
-            return numScopes;
         });
     }
     checkEvent(event, scope, appMapFile, appMapIndex, checkInstance, findings) {
@@ -109,23 +100,16 @@ class RuleChecker {
                     findingEvent.codeObject.location,
                     ...findingEvent.ancestors().map((ancestor) => ancestor.codeObject.location),
                 ].filter(Boolean);
-                const hash = (0, crypto_1.createHash)('sha256');
-                hash.update(findingEvent.hash);
-                hash.update(checkInstance.ruleId);
+                const hashV1 = new hashV1_1.default(checkInstance.ruleId, findingEvent, 
+                // findingEvent gets passed here as a relatedEvent, and if you look at HashV1 it
+                // gets added to the hash again. That's how it worked in V1 so it's here for compatibility.
+                additionalEvents || []);
+                const hashV2 = new hashV2_1.default(checkInstance.ruleId, findingEvent, participatingEvents);
                 const uniqueEvents = new Set();
                 const relatedEvents = [];
-                [findingEvent].concat((additionalEvents || []).map(eventUtil_1.cloneEvent)).forEach((event) => {
-                    if (uniqueEvents.has(event.id)) {
-                        return;
-                    }
-                    uniqueEvents.add(event.id);
-                    relatedEvents.push((0, eventUtil_1.cloneEvent)(event));
-                });
-                // Update event hash with unique hashes of related events
-                new Set(relatedEvents.map((e) => e.hash)).forEach((eventHash) => {
-                    hash.update(eventHash);
-                });
-                Object.values(participatingEvents).forEach((event) => {
+                [findingEvent, ...(additionalEvents || []), ...Object.values(participatingEvents)]
+                    .map(eventUtil_1.cloneEvent)
+                    .forEach((event) => {
                     if (uniqueEvents.has(event.id)) {
                         return;
                     }
@@ -138,7 +122,8 @@ class RuleChecker {
                     ruleId: checkInstance.ruleId,
                     ruleTitle: checkInstance.title,
                     event: (0, eventUtil_1.cloneEvent)(findingEvent),
-                    hash: hash.digest('hex'),
+                    hash: hashV1.digest(),
+                    hash_v2: hashV2.digest(),
                     stack,
                     scope: (0, eventUtil_1.cloneEvent)(scope),
                     message: message || checkInstance.title,

package/built/rules/authzBeforeAuthn.js

@@ -31,6 +31,7 @@ function build() {
                         {
                             event: event.event,
                             message: `${event.event} provides authorization, but the request is not authenticated`,
+                            participatingEvents: { request: rootEvent },
                         },
                     ];
                 }

package/built/rules/jobNotCancelled.js

@@ -18,11 +18,14 @@ function build() {
         const missing = creationEvents.length - cancellationEvents.length;
         if (missing === 0)
             return;
-        return creationEvents.map((jobCreationEvent) => ({
-            event: jobCreationEvent,
-            message: `Job created by ${jobCreationEvent.codeObject.prettyName} was not cancelled when the enclosing transaction rolled back`,
-            participatingEvents: { beginTransaction: event },
-        }));
+        const result = {
+            event: event,
+            message: `${missing} jobs are scheduled but not cancelled in a rolled back transaction`,
+            // if there's a mismatch and there are cancellations we can't tell
+            // for sure which creations they match, so return everything
+            relatedEvents: [...creationEvents, ...cancellationEvents],
+        };
+        return [result];
     }
     return {
         matcher,

package/built/rules/lib/isCommand.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function isCommand(event) {
+    let label;
+    if (event.labels.has('command'))
+        label = 'command';
+    else if (event.labels.has('job'))
+        label = 'job';
+    else if (event.httpServerRequest)
+        label = 'request';
+    return label;
+}
+exports.default = isCommand;

package/built/rules/logoutWithoutSessionReset.js

@@ -28,6 +28,7 @@ function build() {
                         {
                             event: event.event,
                             message: `${event.event} logs out the user, but the HTTP session is not cleared`,
+                            participatingEvents: { request: rootEvent },
                         },
                     ];
                 }

package/built/rules/missingContentType.js

@@ -8,8 +8,10 @@ const parseRuleDescription_1 = __importDefault(require("./lib/parseRuleDescripti
 const isRedirect = (status) => [301, 302, 303, 307, 308].includes(status);
 const hasContent = (status) => status !== 204;
 function build() {
-    function matcher(e) {
-        return (0, openapi_1.rpcRequestForEvent)(e).responseContentType === undefined;
+    function matcher(event) {
+        if ((0, openapi_1.rpcRequestForEvent)(event).responseContentType === undefined) {
+            return `Missing HTTP content type in response to request: ${event.route}`;
+        }
     }
     function where(e) {
         return (!!e.httpServerResponse &&

package/built/rules/nPlusOneQuery.js

@@ -35,6 +35,7 @@ function build(options) {
                 const ancestor = eventsById[parseInt(ancestorId)];
                 const occurranceCount = events.length;
                 if (occurranceCount > options.warningLimit) {
+                    const participatingEvents = { commonAncestor: ancestor };
                     const buildMatchResult = (level) => {
                         return {
                             level: level,
@@ -43,7 +44,7 @@ function build(options) {
                             groupMessage: sql,
                             occurranceCount: occurranceCount,
                             relatedEvents: events.map((e) => e.event),
-                            participatingEvents: { commonAncestor: ancestor },
+                            participatingEvents,
                         };
                     };
                     if (occurranceCount >= options.errorLimit) {

package/built/rules/secretInLog.js

@@ -68,7 +68,7 @@ const findInLog = (event) => {
     if (matches.length > 0) {
         return matches.map((match) => {
             const { pattern, value } = match;
-            const participatingEvents = { logEvent: event };
+            const participatingEvents = {};
             if (match.generatorEvent) {
                 participatingEvents.generatorEvent = match.generatorEvent;
             }
@@ -101,6 +101,7 @@ exports.default = {
     id: 'secret-in-log',
     title: 'Secret in log',
     labels: [Secret, Log],
+    scope: 'root',
     impactDomain: 'Security',
     enumerateScope: true,
     references: {

package/built/rules/slowFunctionCall.js

@@ -30,7 +30,6 @@ function build(options) {
 exports.default = {
     id: 'slow-function-call',
     title: 'Slow function call',
-    scope: 'root',
     impactDomain: 'Performance',
     enumerateScope: true,
     description: (0, parseRuleDescription_1.default)('slowFunctionCall'),

package/built/rules/updateInGetRequest.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const util_1 = require("./lib/util");
 const parseRuleDescription_1 = __importDefault(require("./lib/parseRuleDescription"));
+const assert_1 = __importDefault(require("assert"));
 class Options {
     constructor(queryInclude = [/\binsert\b/i, /\bupdate\b/i], queryExclude = []) {
         this._queryInclude = queryInclude;
@@ -38,7 +39,14 @@ function build(options = new Options()) {
                 !options.queryExclude.some((pattern) => e.sqlQuery.match(pattern)) &&
                 !e.ancestors().some((ancestor) => ancestor.codeObject.labels.has(Audit)) &&
                 hasHttpServerRequest()) {
-                return `Data update performed in ${httpServerRequest.route}: ${e.sqlQuery}`;
+                (0, assert_1.default)(httpServerRequest, 'HTTP server request is undefined');
+                return [
+                    {
+                        event: e,
+                        message: `Data update performed in HTTP request ${httpServerRequest.route}: ${e.sqlQuery}`,
+                        participatingEvents: { request: httpServerRequest },
+                    },
+                ];
             }
         },
         where: (e) => !!e.sqlQuery,

package/built/scope/commandScope.js

@@ -21,15 +21,28 @@ const Command = 'command.perform';
 const Job = 'job.perform';
 class CommandScope extends scopeIterator_1.default {
     *scopes(events) {
+        let found = false;
+        const roots = [];
         for (const event of events) {
+            if (event.isCall() && !event.parent) {
+                roots.push(event);
+            }
             if (event.isCall() &&
                 (event.codeObject.labels.has(Command) ||
                     event.codeObject.labels.has(Job) ||
                     event.httpServerRequest)) {
+                found = true;
                 yield new ScopeImpl(event);
                 this.advanceToReturnEvent(event, events);
             }
         }
+        // If no true command is found, yield all root events.
+        if (!found) {
+            for (let index = 0; index < roots.length; index++) {
+                const event = roots[index];
+                yield new ScopeImpl(event);
+            }
+        }
     }
 }
 exports.default = CommandScope;

package/doc/rules/secret-in-log.md

@@ -8,6 +8,7 @@ impactDomain: Security
 labels:
   - secret
   - log
+scope: root
 ---
 
 Identifies when a known or assumed secret is written to a log. Logs are often transported into other

package/doc/rules/slow-function-call.md

@@ -3,7 +3,6 @@ rule: slow-function-call
 name: Slow function call
 title: Slow function call
 impactDomain: Performance
-scope: root
 ---
 
 Ensures that function elapsed time does not exceed a threshold.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/scanner",
-  "version": "1.62.1",
+  "version": "1.64.0",
   "description": "",
   "bin": "built/cli.js",
   "files": [
@@ -61,7 +61,6 @@
     "@appland/sql-parser": "^1.5.0",
     "@types/cli-progress": "^3.9.2",
     "ajv": "^8.8.2",
-    "ansi-escapes": "^5.0.0",
     "applicationinsights": "^2.1.4",
     "async": "^3.2.3",
     "chalk": "^4.1.2",
@@ -86,16 +85,20 @@
   },
   "pkg": {
     "targets": [
-      "node14-linux-x64",
-      "node14-win-x64",
-      "node14-macos-x64"
+      "node16-linux-x64",
+      "node16-win-x64",
+      "node16-macos-x64",
+      "node16-macos-arm64"
     ],
     "scripts": [
-      "built/scanner/*.js"
+      "built/scanner/*.js",
+      "built/rules/**/*.js"
     ],
     "assets": [
       "built/sampleConfig/*.yml",
-      "built/**/*.json"
+      "built/**/*.json",
+      "package.json",
+      "doc/**/*.md"
     ],
     "outputPath": "dist"
   }