@appland/scanner 1.70.5 → 1.71.1

package/README.md

@@ -66,7 +66,7 @@ documentation for each rule to see it's pattern filters and other configurable p
 To enable de-duplication of findings, a hash is calculated for each finding. The hash is the
 `sha256` digest of a canonical content string for the finding. The canonical content string combines
 stable data from the finding, such as the rule id, normalized event, etc. See
-[appmap-js/packages/models/src/event/hash.js](https://github.com/applandinc/appmap-js/blob/959a8c93c9be37d40a8f4a0e7d44ee211730641e/packages/models/src/event/hash.js)
+[appmap-js/packages/models/src/event/hash.js](https://github.com/getappmap/appmap-js/blob/959a8c93c9be37d40a8f4a0e7d44ee211730641e/packages/models/src/event/hash.js)
 for details.
 
 ## Findings output file
@@ -102,7 +102,7 @@ _Example_
     "clients": [
       {
         "name": "appmap",
-        "url": "https://github.com/applandinc/appmap-ruby",
+        "url": "https://github.com/getappmap/appmap-ruby",
         "version": "0.70.2"
       }
     ],
@@ -118,12 +118,12 @@ _Example_
     ],
     "git": [
       {
-        "repository": "git@github.com:applandinc/appmap-server.git",
+        "repository": "git@github.com:getappmap/appmap-server.git",
         "branch": "master",
         "commit": "3b028018ec1f84e2c351d01d1dac45aeeae887b6"
       },
       {
-        "repository": "git@github.com:applandinc/appmap-server.git",
+        "repository": "git@github.com:getappmap/appmap-server.git",
         "branch": "master",
         "commit": "3b028018ec1f84e2c351d01d1dac45aeeae887b6",
         "status": [
@@ -200,11 +200,11 @@ _Example_
   "tmp/appmap/rspec/API_APIKeysController_create_a_new_api_key.appmap.json": {
     "client": {
       "name": "appmap",
-      "url": "https://github.com/applandinc/appmap-ruby",
+      "url": "https://github.com/getappmap/appmap-ruby",
       "version": "0.70.1"
     },
     "git": {
-      "repository": "git@github.com:applandinc/appmap-server.git",
+      "repository": "git@github.com:getappmap/appmap-server.git",
       "branch": "master",
       "commit": "3b028018ec1f84e2c351d01d1dac45aeeae887b6"
     },

package/built/cli/resolveAppId.js

@@ -9,11 +9,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const src_1 = require("@appland/client/dist/src");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const js_yaml_1 = require("js-yaml");
 const path_1 = require("path");
-const exists_1 = require("../integration/appland/app/exists");
 const errors_1 = require("../errors");
 function resolveAppId(appIdArg, appMapDir) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -44,7 +44,7 @@ function default_1(appIdArg, appMapDir) {
         const appId = yield resolveAppId(appIdArg, appMapDir);
         if (!appId)
             throw new errors_1.ValidationError('App was not provided and could not be resolved');
-        const appExists = yield (0, exists_1.exists)(appId);
+        const appExists = yield new src_1.App(appId).exists();
         if (!appExists) {
             throw new errors_1.ValidationError(`App "${appId}" is not valid or does not exist.\nPlease fix the app name in the appmap.yml file, or override it with the --app option.`);
         }

package/built/cli/scan/scanner.js

@@ -14,7 +14,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const src_1 = require("@appland/client/dist/src");
 const configurationProvider_1 = require("../../configuration/configurationProvider");
-const listFindingStatus_1 = __importDefault(require("../../integration/appland/app/listFindingStatus"));
 const resolveAppId_1 = __importDefault(require("../resolveAppId"));
 const scan_1 = __importDefault(require("../scan"));
 const scanResults_1 = require("../../report/scanResults");
@@ -47,7 +46,7 @@ class ServerIntegratedScanner extends ScannerBase {
     fetchFindingStatus(appIdArg, appMapDir) {
         return __awaiter(this, void 0, void 0, function* () {
             const appId = yield (0, resolveAppId_1.default)(appIdArg, appMapDir);
-            return yield (0, listFindingStatus_1.default)(appId);
+            return yield new src_1.App(appId).listFindingStatus();
         });
     }
 }

package/built/cli/scan/watchScan.js

@@ -40,14 +40,14 @@ const promises_1 = require("fs/promises");
 const chokidar = __importStar(require("chokidar"));
 const formatReport_1 = require("./formatReport");
 const scanner_1 = __importDefault(require("./scanner"));
-const fs_1 = require("fs");
-const util_1 = require("util");
 const configurationProvider_1 = require("../../configuration/configurationProvider");
 const assert_1 = __importDefault(require("assert"));
 const path_1 = __importDefault(require("path"));
+const async_1 = require("async");
 class Watcher {
     constructor(options) {
         this.options = options;
+        this.queue = (0, async_1.queue)(this.scan.bind(this), 2);
     }
     watch() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -71,10 +71,9 @@ class Watcher {
                 interval: 1000,
                 persistent: false,
             });
-            for (const ch of [this.appmapWatcher, this.appmapPoller]) {
-                ch.on('add', (filePath) => this.scan(filePath));
-                ch.on('change', (filePath) => this.scan(filePath));
-            }
+            const enqueue = this.enqueue.bind(this);
+            for (const ch of [this.appmapWatcher, this.appmapPoller])
+                ch.on('add', enqueue).on('change', enqueue);
         });
     }
     close() {
@@ -87,15 +86,24 @@ class Watcher {
             }));
         });
     }
-    scan(fileName) {
+    enqueue(mtimePath) {
+        if ([...this.queue].includes(mtimePath))
+            return;
+        this.queue.push(mtimePath);
+    }
+    scan(mtimePath) {
         return __awaiter(this, void 0, void 0, function* () {
             (0, assert_1.default)(this.config, `config should always be loaded before appmapWatcher triggers a scan`);
-            const pathTokens = fileName.split('/');
-            const appmapDir = pathTokens.slice(0, pathTokens.length - 1).join('/');
-            const appmapFile = [appmapDir, 'appmap.json'].join('.');
-            const reportFile = [appmapDir, 'appmap-findings.json'].join('/');
-            if (!(yield (0, util_1.promisify)(fs_1.exists)(appmapFile)))
+            (0, assert_1.default)(this.configTime);
+            const appmapFile = mtimePath.replace(/\/mtime$/, '.appmap.json');
+            const reportFile = mtimePath.replace(/mtime$/, 'appmap-findings.json');
+            const [appmapStats, reportStats] = yield Promise.all([appmapFile, reportFile].map((f) => (0, promises_1.stat)(f).catch(() => null)));
+            if (!appmapStats)
                 return;
+            if (reportStats &&
+                reportStats.mtimeMs > appmapStats.mtimeMs &&
+                reportStats.mtimeMs > this.configTime)
+                return; // report is up to date
             const scanner = yield (0, scanner_1.default)(true, this.config, [appmapFile]);
             const rawScanResults = yield scanner.scan();
             // Always report the raw data
@@ -105,6 +113,7 @@ class Watcher {
     reloadConfig() {
         return __awaiter(this, void 0, void 0, function* () {
             this.config = yield (0, configurationProvider_1.parseConfigFile)(this.options.configFile);
+            this.configTime = (yield (0, promises_1.stat)(this.options.configFile)).mtimeMs;
         });
     }
 }

package/built/cli/upload.js

@@ -11,10 +11,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 const async_1 = require("async");
 const promises_1 = require("fs/promises");
+const src_1 = require("@appland/client/dist/src");
 const util_1 = require("../rules/lib/util");
-const create_1 = require("../integration/appland/appMap/create");
-const create_2 = require("../integration/appland/mapset/create");
-const create_3 = require("../integration/appland/scannerJob/create");
+const create_1 = require("../integration/appland/scannerJob/create");
 const vars_1 = require("../integration/vars");
 const promises_2 = require("fs/promises");
 const path_1 = require("path");
@@ -79,7 +78,7 @@ function create(scanResults, appId, appMapDir, mergeKey, mapsetOptions = {}, ret
                     commitCount[commit] || (commitCount[commit] = 1);
                     commitCount[commit] += 1;
                 }
-                return (0, create_1.create)(buffer, Object.assign(retryOptions, Object.assign(Object.assign({}, createAppMapOptions), { metadata })));
+                return src_1.AppMap.create(buffer, Object.assign(retryOptions, Object.assign(Object.assign({}, createAppMapOptions), { metadata })));
             })
                 .then((appMap) => {
                 if (appMap) {
@@ -104,9 +103,9 @@ function create(scanResults, appId, appMapDir, mergeKey, mapsetOptions = {}, ret
         };
         mapsetOptions.branch || (mapsetOptions.branch = (0, vars_1.branch)() || mostFrequent(branchCount));
         mapsetOptions.commit || (mapsetOptions.commit = (0, vars_1.sha)() || mostFrequent(commitCount));
-        const mapset = yield (0, create_2.create)(appId, Object.values(appMapUUIDByFileName), mapsetOptions, retryOptions);
+        const mapset = yield src_1.Mapset.create(appId, Object.values(appMapUUIDByFileName), mapsetOptions, retryOptions);
         console.warn('Uploading findings');
-        return (0, create_3.create)(scanResults, mapset.id, appMapUUIDByFileName, { mergeKey }, retryOptions);
+        return (0, create_1.create)(scanResults, mapset.id, appMapUUIDByFileName, { mergeKey }, retryOptions);
     });
 }
 exports.default = create;

package/built/rules/too-many-joins/rule.js

@@ -13,7 +13,7 @@ function rule(options) {
         (0, assert_1.default)(table.variant === 'table');
         return !excludeTables.find((filter) => filter(table.name));
     }
-    // TODO: clean up (https://github.com/applandinc/scanner/issues/43)
+    // TODO: clean up (https://github.com/getappmap/scanner/issues/43)
     function matcher(command, appMapIndex, eventFilter) {
         const joinCount = {};
         for (const sqlEvent of (0, database_1.sqlStrings)(command, appMapIndex, eventFilter)) {

package/doc/architecture.md

@@ -1,6 +1,6 @@
 ## Scanner architecture
 
-See [@appland/models source code](https://github.com/applandinc/appmap-js/tree/main/packages/models)
+See [@appland/models source code](https://github.com/getappmap/appmap-js/tree/main/packages/models)
 for the JS API to AppMap data.
 
 ## Assertions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/scanner",
-  "version": "1.70.5",
+  "version": "1.71.1",
   "description": "",
   "bin": "built/cli.js",
   "files": [
@@ -67,11 +67,10 @@
     "async": "^3.2.3",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
-    "chokidar": "applandinc/chokidar#fix/new-file-new-directory-race-on-linux",
+    "chokidar": "^3.5.1",
     "cli-progress": "^3.11.0",
     "conf": "^10.0.2",
     "crypto-js": "^4.0.0",
-    "form-data": "^4.0.0",
     "glob": "7.2.3",
     "inquirer": "^8.1.2",
     "js-yaml": "^4.1.0",

package/built/integration/appland/app/exists.js

@@ -1,55 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.exists = void 0;
-const src_1 = require("@appland/client/dist/src");
-const util_1 = require("../../../rules/lib/util");
-const retry_1 = __importDefault(require("../retry"));
-function exists(appId, retryOptions = {}) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const commandDescription = `Checking if app ${appId} exists`;
-        if ((0, util_1.verbose)())
-            console.log(commandDescription);
-        const retrier = (0, retry_1.default)(commandDescription, retryOptions, makeRequest);
-        function makeRequest() {
-            return __awaiter(this, void 0, void 0, function* () {
-                const requestPath = ['api', appId].join('/');
-                const request = yield (0, src_1.buildRequest)(requestPath);
-                return new Promise((resolve, reject) => {
-                    const req = request.requestFunction(request.url, {
-                        method: 'HEAD',
-                        headers: request.headers,
-                    }, resolve);
-                    req.on('error', (0, src_1.retryOnError)(retrier, resolve, reject));
-                    req.end();
-                }).then((0, src_1.retryOn503)(retrier));
-            });
-        }
-        return makeRequest().then((response) => __awaiter(this, void 0, void 0, function* () {
-            if ((0, util_1.verbose)())
-                console.log(`${commandDescription}: statusCode=${response.statusCode}`);
-            if (!response.statusCode) {
-                throw new Error('No status code was provided by the server');
-            }
-            if (response.statusCode === 404) {
-                return false;
-            }
-            else if (response.statusCode < 300) {
-                return true;
-            }
-            throw new Error(`Unexpected status code: ${response.statusCode}`);
-        }));
-    });
-}
-exports.exists = exists;

package/built/integration/appland/app/listFindingStatus.js

@@ -1,18 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const src_1 = require("@appland/client/dist/src");
-function listFindingStatus(appId) {
-    return __awaiter(this, void 0, void 0, function* () {
-        return yield new src_1.App(appId).listFindingStatus();
-    });
-}
-exports.default = listFindingStatus;

package/built/integration/appland/appMap/create.js

@@ -1,60 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.create = void 0;
-const src_1 = require("@appland/client/dist/src");
-const form_data_1 = __importDefault(require("form-data"));
-const retry_1 = __importDefault(require("../retry"));
-function create(data, options, retryOptions = {}) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const retrier = (0, retry_1.default)(`Upload AppMap`, retryOptions, makeRequest);
-        function makeRequest() {
-            return __awaiter(this, void 0, void 0, function* () {
-                const form = new form_data_1.default();
-                form.append('data', data.toString());
-                if (options.metadata) {
-                    form.append('metadata', JSON.stringify(options.metadata));
-                }
-                if (options.app) {
-                    form.append('app', options.app);
-                }
-                const request = yield (0, src_1.buildRequest)('api/appmaps');
-                return new Promise((resolve, reject) => {
-                    const req = request.requestFunction(request.url, {
-                        method: 'POST',
-                        headers: Object.assign(Object.assign({}, request.headers), form.getHeaders()),
-                    }, resolve);
-                    req.on('error', (0, src_1.retryOnError)(retrier, resolve, reject));
-                    form.pipe(req);
-                }).then((0, src_1.retryOn503)(retrier));
-            });
-        }
-        return makeRequest()
-            .then(src_1.handleError)
-            .then((response) => {
-            return new Promise((resolve, reject) => {
-                const responseData = [];
-                response
-                    .on('data', (chunk) => {
-                    responseData.push(Buffer.from(chunk));
-                })
-                    .on('end', () => {
-                    resolve(JSON.parse(Buffer.concat(responseData).toString()));
-                })
-                    .on('error', reject);
-            });
-        });
-    });
-}
-exports.create = create;

package/built/integration/appland/mapset/create.js

@@ -1,56 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.create = void 0;
-const src_1 = require("@appland/client/dist/src");
-const util_1 = require("../../../rules/lib/util");
-const retry_1 = __importDefault(require("../retry"));
-function create(appId, appMapIds, options, retryOptions = {}) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if ((0, util_1.verbose)())
-            console.log(`Creating mapset in app ${appId} with ${appMapIds.length} AppMaps`);
-        const retrier = (0, retry_1.default)(`Create Mapset`, retryOptions, makeRequest);
-        function makeRequest() {
-            return __awaiter(this, void 0, void 0, function* () {
-                const payload = JSON.stringify(Object.assign({ app: appId, appmaps: appMapIds }, options));
-                const request = yield (0, src_1.buildRequest)('api/mapsets');
-                return new Promise((resolve, reject) => {
-                    const req = request.requestFunction(request.url, {
-                        method: 'POST',
-                        headers: Object.assign({ 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }, request.headers),
-                    }, resolve);
-                    req.on('error', (0, src_1.retryOnError)(retrier, resolve, reject));
-                    req.write(payload);
-                    req.end();
-                }).then((0, src_1.retryOn503)(retrier));
-            });
-        }
-        return makeRequest()
-            .then(src_1.handleError)
-            .then((response) => {
-            return new Promise((resolve, reject) => {
-                const responseData = [];
-                response
-                    .on('data', (chunk) => {
-                    responseData.push(Buffer.from(chunk));
-                })
-                    .on('end', () => {
-                    resolve(JSON.parse(Buffer.concat(responseData).toString()));
-                })
-                    .on('error', reject);
-            });
-        });
-    });
-}
-exports.create = create;