@appland/scanner 1.70.0 → 1.70.2

package/built/cli/scan/scanner.js

@@ -35,10 +35,10 @@ class ScannerBase {
         this.configuration = configuration;
         this.files = files;
     }
-    scan() {
+    scan(skipErrors = false) {
         return __awaiter(this, void 0, void 0, function* () {
             const checks = yield (0, configurationProvider_1.loadConfig)(this.configuration);
-            const { appMapMetadata, findings } = yield (0, scan_1.default)(this.files, checks);
+            const { appMapMetadata, findings } = yield (0, scan_1.default)(this.files, checks, skipErrors);
             return new scanResults_1.ScanResults(this.configuration, appMapMetadata, findings, checks);
         });
     }

package/built/cli/scan/singleScan.js

@@ -25,6 +25,7 @@ const validateFile_1 = __importDefault(require("../validateFile"));
 function singleScan(options) {
     return __awaiter(this, void 0, void 0, function* () {
         const { appmapFile, appmapDir, configuration, reportAllFindings, appId, ide, reportFile } = options;
+        const skipErrors = appmapDir !== undefined;
         const files = yield (0, util_1.collectAppMapFiles)(appmapFile, appmapDir);
         yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () { return (0, validateFile_1.default)('file', file); })));
         const scanner = yield (0, scanner_1.default)(reportAllFindings, configuration, files).catch((error) => {
@@ -32,7 +33,7 @@ function singleScan(options) {
         });
         const startTime = Date.now();
         const [rawScanResults, findingStatuses] = yield Promise.all([
-            scanner.scan(),
+            scanner.scan(skipErrors),
             scanner.fetchFindingStatus(appId, appmapDir),
         ]);
         // Always report the raw data

package/built/cli/scan.js

@@ -18,6 +18,7 @@ const promises_1 = require("fs/promises");
 const models_1 = require("@appland/models");
 const ruleChecker_1 = __importDefault(require("../ruleChecker"));
 const appMapIndex_1 = __importDefault(require("../appMapIndex"));
+const node_assert_1 = __importDefault(require("node:assert"));
 function batch(items, size, process) {
     return __awaiter(this, void 0, void 0, function* () {
         const left = [...items];
@@ -25,7 +26,37 @@ function batch(items, size, process) {
             yield Promise.all(left.splice(0, size).map(process));
     });
 }
-function scan(files, checks) {
+class Progress {
+    constructor(numFiles, numChecks) {
+        this.numFiles = numFiles;
+        this.numChecks = numChecks;
+        this.checks = 0;
+        if (process.stdout.isTTY)
+            this.bar = new cli_progress_1.default.SingleBar({ format: `Scanning [{bar}] {percentage}% | {value}/{total}` }, cli_progress_1.default.Presets.shades_classic);
+        else {
+            this.start = this.check = this.file = this.stop = () => { };
+        }
+    }
+    start() {
+        var _a;
+        (_a = this.bar) === null || _a === void 0 ? void 0 : _a.start(this.numFiles * this.numChecks, 0);
+    }
+    check() {
+        var _a;
+        this.checks += 1;
+        (_a = this.bar) === null || _a === void 0 ? void 0 : _a.increment();
+    }
+    file() {
+        var _a;
+        (_a = this.bar) === null || _a === void 0 ? void 0 : _a.increment(this.numChecks - this.checks);
+        this.checks = 0;
+    }
+    stop() {
+        var _a;
+        (_a = this.bar) === null || _a === void 0 ? void 0 : _a.stop();
+    }
+}
+function scan(files, checks, skipErrors = true) {
     return __awaiter(this, void 0, void 0, function* () {
         // TODO: Improve this by respecting .gitignore, or similar.
         // For now, this addresses the main problem of encountering appmap-js and its appmap.json files
@@ -34,33 +65,37 @@ function scan(files, checks) {
         const checker = new ruleChecker_1.default();
         const appMapMetadata = {};
         const findings = [];
-        function newProgress() {
-            if (process.stdout.isTTY) {
-                return new cli_progress_1.default.SingleBar({ format: `Scanning [{bar}] {percentage}% | {value}/{total}` }, cli_progress_1.default.Presets.shades_classic);
-            }
-            return {
-                increment: () => { },
-                start: () => { },
-                stop: () => { },
-            };
-        }
-        const progress = newProgress();
-        progress.start(files.length * checks.length, 0);
+        const progress = new Progress(files.length, checks.length);
+        let lastError = null;
+        let anySuccess = false;
         yield batch(files, 2, (file) => __awaiter(this, void 0, void 0, function* () {
-            const appMapData = yield (0, promises_1.readFile)(file, 'utf8');
-            const appMap = (0, models_1.buildAppMap)(appMapData).normalize().build();
-            const appMapIndex = new appMapIndex_1.default(appMap);
-            appMapMetadata[file] = appMap.metadata;
-            yield Promise.all(checks.map((check) => __awaiter(this, void 0, void 0, function* () {
-                const matchCount = findings.length;
-                yield checker.check(file, appMapIndex, check, findings);
-                progress.increment();
-                const newMatches = findings.slice(matchCount, findings.length);
-                newMatches.forEach((match) => (match.appMapFile = file));
-            })));
-            return null;
+            try {
+                console.log(`scanning ${file}`);
+                const appMapData = yield (0, promises_1.readFile)(file, 'utf8');
+                const appMap = (0, models_1.buildAppMap)(appMapData).normalize().build();
+                const appMapIndex = new appMapIndex_1.default(appMap);
+                appMapMetadata[file] = appMap.metadata;
+                yield Promise.all(checks.map((check) => __awaiter(this, void 0, void 0, function* () {
+                    const matchCount = findings.length;
+                    yield checker.check(file, appMapIndex, check, findings);
+                    progress.check();
+                    const newMatches = findings.slice(matchCount, findings.length);
+                    newMatches.forEach((match) => (match.appMapFile = file));
+                })));
+                anySuccess = true;
+            }
+            catch (error) {
+                (0, node_assert_1.default)(error instanceof Error);
+                lastError = new Error(`Error processing "${file}"`, { cause: error });
+                if (!skipErrors)
+                    throw lastError;
+                console.warn(lastError);
+            }
+            progress.file();
         }));
         progress.stop();
+        if (!anySuccess && lastError)
+            throw lastError;
         return { appMapMetadata, findings };
     });
 }

package/doc/architecture.md

@@ -1,48 +1,63 @@
 ## Scanner architecture
 
-See [@appland/models source code](https://github.com/applandinc/appmap-js/tree/main/packages/models) for the JS API to AppMap data.
+See [@appland/models source code](https://github.com/applandinc/appmap-js/tree/main/packages/models)
+for the JS API to AppMap data.
 
 ## Assertions
 
-An Assertion tests each configured AppMap event to see if it matches some condition. The test is applied by a `matcher` fnuction.
+An Assertion tests each configured AppMap event to see if it matches some condition. The test is
+applied by a `matcher` fnuction.
 
-If there is a match, the assertion returns a Finding. A Finding contains the type of check, the event, and a descriptive message. Supporting (related) events may also be reported.
+If there is a match, the assertion returns a Finding. A Finding contains the type of check, the
+event, and a descriptive message. Supporting (related) events may also be reported.
 
 ## Scopes
 
-Each Assertion declares a Scope. The Scope is the set of events that will be checked by an instance of the Assertion object. An Assertion can use a narrower scope to help avoid giving false positives. For example, consider an Assertion that looks for "too many SQL queries". The Assertion only wants to count SQL queries within the Scope of a single command - not the entire AppMap.
-
+Each Assertion declares a Scope. The Scope is the set of events that will be checked by an instance
+of the Assertion object. An Assertion can use a narrower scope to help avoid giving false positives.
+For example, consider an Assertion that looks for "too many SQL queries". The Assertion only wants
+to count SQL queries within the Scope of a single command - not the entire AppMap.
 
 Scope examples (roughly ordered from broadest to narrowest):
 
-* `all` All events in the AppMap will be processed by the same Assertion instance.
-* `root` A new Assertion instance is created for each root event.
-* `command` A new Assertion instance is created for each HTTP server request, and for each event that is not a descendant of an HTTP server request AND has the label `command` or `job`.
-* `http_server_request` A new Assertion instance is created for each HTTP server request.
-* `transaction` A new Assertion instance is created for each database transaction in the AppMap.
+- `all` All events in the AppMap will be processed by the same Assertion instance.
+- `root` A new Assertion instance is created for each root event.
+- `command` A new Assertion instance is created for each HTTP server request, and for each event
+  that is not a descendant of an HTTP server request AND has the label `command` or `job`.
+- `http_server_request` A new Assertion instance is created for each HTTP server request.
+- `transaction` A new Assertion instance is created for each database transaction in the AppMap.
 
 ## Event filters
 
 Assertions use Event filters to choose which events are processed by the `matcher` function.
 
-Event filters include the `where`, `include` and `exclude` conditions. Events must match the `where` and `include` conditions, and must not match the `exclude` condition. The `where` condition is built into the Assertion. The `include` and `exclude` conditions are blank, and exist to be customized by the user.
+Event filters include the `where`, `include` and `exclude` conditions. Events must match the `where`
+and `include` conditions, and must not match the `exclude` condition. The `where` condition is built
+into the Assertion. The `include` and `exclude` conditions are blank, and exist to be customized by
+the user.
 
 ## Examples
 
 ### HTTP 500
 
-`http-500` assertion is a simple example. It specifies the `http_server_request` scope - so that each HTTP server request is processed by a separate Assertion. 
+`http-500` assertion is a simple example. It specifies the `http_server_request` scope - so that
+each HTTP server request is processed by a separate Assertion.
 
-The `where` condition filter out events that don't have an `http_server_response` - for example, if the server process was hard-killed in the middle of processing.
+The `where` condition filter out events that don't have an `http_server_response` - for example, if
+the server process was hard-killed in the middle of processing.
 
 The `matcher` function returns true if the HTTP status code is between 500 and 599.
 
 ### Insecure compare
 
-`insecure-compare` operates on the `all` scope - it looks for insecure compare across the entire AppMap.
+`insecure-compare` operates on the `all` scope - it looks for insecure compare across the entire
+AppMap.
 
-The `where` clause selects events that are labeled `string.equals` or `secret`. The `secret` label is used to build a Set of all the secrets that are generated/returned by function events in the AppMap. When a `string.equals` function is encountered, the assertion returns true if:
+The `where` clause selects events that are labeled `string.equals` or `secret`. The `secret` label
+is used to build a Set of all the secrets that are generated/returned by function events in the
+AppMap. When a `string.equals` function is encountered, the assertion returns true if:
 
 1. The function has a receiver value and one parameter.
 2. Both the receiver value and the parameter value are not BCrypted-strings.
-3. Both the receiver value and the parameter value are either (a) a known secret or (b) match a secret regexp
+3. Both the receiver value and the parameter value are either (a) a known secret or (b) match a
+   secret regexp

package/doc/labels/deserialize.unsafe.md

@@ -11,4 +11,4 @@ Indicates that a function does not guarantee safe deserialization.
 - Ruby [YAML.unsafe_load](https://docs.ruby-lang.org/en/3.0/Psych.html#method-c-unsafe_load)
 - Ruby [Marshal.load](https://docs.ruby-lang.org/en/3.0/Marshal.html#method-c-load)
 - Java
-  [javax.jms.ObjectMessage#getObject](https://docs.oracle.com/javaee/6/api/javax/jms/ObjectMessage.html#getObject())
+  [javax.jms.ObjectMessage#getObject](<https://docs.oracle.com/javaee/6/api/javax/jms/ObjectMessage.html#getObject()>)

package/doc/rules/deserialization-of-untrusted-data.md

@@ -18,27 +18,27 @@ data comes from an untrusted source and hasn't passed through a sanitization mec
 
 ### Rule logic
 
-Finds all events labeled `deserialize.unsafe` that receive tainted data (as
-determined by object identity or string value) as an input.
+Finds all events labeled `deserialize.unsafe` that receive tainted data (as determined by object
+identity or string value) as an input.
 
 For each of these events; checks if all the inputs have been sanitized.
 
-Data that has been passed to a function labeled `deserialize.sanitize` is
-assumed to be sanitized from this point onwards. Such a function could either
-check the value is sanitized (note no verification is currently done to ensure
-this result is checked) or return the transformed value after any necessary sanitization.
+Data that has been passed to a function labeled `deserialize.sanitize` is assumed to be sanitized
+from this point onwards. Such a function could either check the value is sanitized (note no
+verification is currently done to ensure this result is checked) or return the transformed value
+after any necessary sanitization.
 
-Data passed to a function labeled `deserialized.safe` is considered in all
-functions called by it (down the callstack). Functions that first sanitize data
-and then use an unsafe deserialization function should carry this label.
+Data passed to a function labeled `deserialized.safe` is considered in all functions called by it
+(down the callstack). Functions that first sanitize data and then use an unsafe deserialization
+function should carry this label.
 
-The set of tracked tainted data initially includes the HTTP message parameters
-and is expanded to include any non-primitive (ie. longer than 5 characters)
-observed outputs of functions that consume tainted data.
+The set of tracked tainted data initially includes the HTTP message parameters and is expanded to
+include any non-primitive (ie. longer than 5 characters) observed outputs of functions that consume
+tainted data.
 
-The reliability of this rule now depends on completeness of the AppMap.
-If there is a data transformation that is	not captured it's invisible to the
-rule and will result in failure to associate it with the tracked untrusted data.
+The reliability of this rule now depends on completeness of the AppMap. If there is a data
+transformation that is not captured it's invisible to the rule and will result in failure to
+associate it with the tracked untrusted data.
 
 ### Notes
 
@@ -47,12 +47,13 @@ that executes code shortly after deserialization.
 
 ### Resolution
 
-Consider if the library you're using offers a safe deserialization function variant that you can
-use instead. Using unsafe functions is only rarely needed and typically requires a good reason.
+Consider if the library you're using offers a safe deserialization function variant that you can use
+instead. Using unsafe functions is only rarely needed and typically requires a good reason.
 
 If you need to use the unsafe function, make sure you're able to handle unexpected input safely.
-Sanitize the data thoroughly first; label the sanitization function with `deserialize.sanitize` label
-or wrap the whole sanitization and deserialization logic in a function labeled `deserialize.safe`.
+Sanitize the data thoroughly first; label the sanitization function with `deserialize.sanitize`
+label or wrap the whole sanitization and deserialization logic in a function labeled
+`deserialize.safe`.
 
 If you need to deserialize untrusted data, JSON is often a good choice as it is only capable of
 returning ‘primitive’ types such as strings, arrays, hashes, numbers and nil. If you need to

package/doc/rules/n-plus-one-query.md

@@ -8,7 +8,6 @@ impactDomain: Performance
 scope: command
 ---
 
-
 Finds occurrences of a query being repeated within a loop.
 
 ### Rule logic

package/doc/rules/query-from-invalid-package.md

@@ -29,8 +29,8 @@ the allowed packages.
 
 - `allowedPackages: `[MatchPatternConfig](/docs/analysis/match-pattern-config.html)`[]`. Packages
   which are allowed to make queries. Required.
-- `allowedQueries: `[MatchPatternConfig](/docs/analysis/match-pattern-config.html)`[]`. Queries which
-  are allowed from anywhere. Default:
+- `allowedQueries: `[MatchPatternConfig](/docs/analysis/match-pattern-config.html)`[]`. Queries
+  which are allowed from anywhere. Default:
   `[/\bBEGIN\b/i, /\bCOMMIT\b/i, /\bROLLBACK\b/i, /\bRELEASE\b/i, /\bSAVEPOINT\b/i]`.
 
 ### Examples

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/scanner",
-  "version": "1.70.0",
+  "version": "1.70.2",
   "description": "",
   "bin": "built/cli.js",
   "files": [
@@ -46,9 +46,10 @@
     "nock": "^13.2.2",
     "openapi-types": "^9.3.0",
     "pkg": "^5.5.2",
-    "prettier": "^2.3.2",
+    "prettier": "^2.7.1",
     "semantic-release": "^19.0.2",
     "sinon": "^13.0.1",
+    "tmp-promise": "^3.0.3",
     "ts-jest": "^27.1.4",
     "ts-json-schema-generator": "^0.97.0",
     "ts-node": "^10.2.1",
@@ -76,6 +77,7 @@
     "minimatch": "^3.0.4",
     "octokat": "^0.10.0",
     "openapi-diff": "^0.23.5",
+    "ora": "~5",
     "pretty-format": "^27.4.6",
     "read-pkg-up": "^7.0.1",
     "supports-hyperlinks": "^2.2.0",