@static-suite/static-suite-data-server 1.0.4 → 1.0.6

package/dist/http/routes/diff.d.ts

@@ -0,0 +1,7 @@
+import { Request, Response } from 'express';
+declare const diffIndex: (req: Request, res: Response) => void;
+declare const diffIncremental: (req: Request, res: Response) => void;
+declare const diffFull: (req: Request, res: Response) => void;
+declare const diffReset: (req: Request, res: Response) => void;
+export { diffIndex, diffIncremental, diffFull, diffReset };
+//# sourceMappingURL=diff.d.ts.map

package/dist/http/routes/diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../../src/http/routes/diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAI5C,QAAA,MAAM,SAAS,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAShD,CAAC;AAeF,QAAA,MAAM,eAAe,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAEtD,CAAC;AAEF,QAAA,MAAM,QAAQ,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAE/C,CAAC;AAEF,QAAA,MAAM,SAAS,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAQhD,CAAC;AAEF,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC"}

package/dist/http/routes/diff.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diffReset = exports.diffFull = exports.diffIncremental = exports.diffIndex = void 0;
+const diff_1 = require("../../lib/store/diff");
+const object_1 = require("../../lib/utils/object");
+const diffIndex = (req, res) => {
+    res.render('diffIndex', {
+        links: {
+            '/diff/incremental': 'Execute an incremental dump',
+            '/diff/full': 'Execute a full dump',
+            '/diff/reset': 'Clears intermediate changes tracked by dependency manager',
+        },
+    });
+};
+exports.diffIndex = diffIndex;
+const diffAction = (req, res, incremental) => {
+    const diff = diff_1.diffManager.getDiff({ incremental });
+    const diffAsJson = (0, object_1.jsonify)(diff);
+    res.status(200);
+    res.set({ 'Content-Type': 'application/json' });
+    res.send(diffAsJson);
+};
+const diffIncremental = (req, res) => {
+    diffAction(req, res, true);
+};
+exports.diffIncremental = diffIncremental;
+const diffFull = (req, res) => {
+    diffAction(req, res, false);
+};
+exports.diffFull = diffFull;
+const diffReset = (req, res) => {
+    const args = req.query;
+    if (args?.uniqueId) {
+        diff_1.diffManager.reset(args.uniqueId);
+    }
+    res.status(200);
+    res.set({ 'Content-Type': 'application/json' });
+    res.send({ status: 'done' });
+};
+exports.diffReset = diffReset;

package/dist/http/routes/home.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"home.d.ts","sourceRoot":"","sources":["../../../src/http/routes/home.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE5C,eAAO,MAAM,IAAI,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAalD,CAAC"}
+{"version":3,"file":"home.d.ts","sourceRoot":"","sources":["../../../src/http/routes/home.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE5C,eAAO,MAAM,IAAI,GAAI,KAAK,OAAO,EAAE,KAAK,QAAQ,KAAG,IAclD,CAAC"}

package/dist/http/routes/home.js

@@ -6,6 +6,7 @@ const home = (req, res) => {
         links: {
             '/data': 'Browse all data loaded into the server',
             '/dependency': 'Dependency data',
+            '/diff': 'Diff operations',
             '/dump': 'Dump operations',
             '/query': 'List of available queries',
             '/task': 'List of available tasks',

package/dist/http/routes/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/http/routes/index.ts"],"names":[],"mappings":"AAmCA,QAAA,MAAM,MAAM,4CAAW,CAAC;AAiCxB,eAAe,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/http/routes/index.ts"],"names":[],"mappings":"AAoCA,QAAA,MAAM,MAAM,4CAAW,CAAC;AAqCxB,eAAe,MAAM,CAAC"}

package/dist/http/routes/index.js

@@ -13,6 +13,7 @@ const dependency_1 = require("./dependency");
 const indexUrl_1 = require("./indexUrl");
 const indexUUID_1 = require("./indexUUID");
 const subset_1 = require("./subset");
+const diff_1 = require("./diff");
 const routes = (0, express_1.Router)();
 routes.get('/', home_1.home);
 routes.get('/docs', docs_1.docs);
@@ -28,6 +29,10 @@ routes.get('/task', task_1.taskIndex);
 routes.get('/task/:taskId', task_1.runTask);
 routes.get('/cache', cache_1.cacheIndex);
 routes.get('/cache/:cacheBinId/clear', cache_1.cacheClear);
+routes.get('/diff', diff_1.diffIndex);
+routes.get('/diff/incremental', diff_1.diffIncremental);
+routes.get('/diff/full', diff_1.diffFull);
+routes.get('/diff/reset', diff_1.diffReset);
 routes.get('/dump', dump_1.dumpIndex);
 routes.get('/dump/incremental', dump_1.dumpIncremental);
 routes.get('/dump/full', dump_1.dumpFull);

package/dist/http/views/diffIndex.pug

@@ -0,0 +1,14 @@
+extends includes/layout.pug
+
+block vars
+  - var title = "Diff | Static Suite Data Server"
+
+block content
+  h1(class="title is-1")="Diff"
+  table(class="table is-fullwidth is-hoverable")
+    tbody
+      each description, link in links
+        tr
+          td
+            a(href=link)= link
+          td=description

package/dist/lib/dataServer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataServer.d.ts","sourceRoot":"","sources":["../../src/lib/dataServer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAKhE,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,WAAW,EACX,uBAAuB,EACvB,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,OAAO,EACP,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,QAAA,MAAM,UAAU;IACd;;;;;;OAMG;oBACa,qBAAqB,KAAG,gBAAgB;CA6BzD,CAAC;AACF,OAAO,EACL,UAAU,EACV,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,WAAW,EACX,uBAAuB,EACvB,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,UAAU,EACV,WAAW,GACZ,CAAC"}
+{"version":3,"file":"dataServer.d.ts","sourceRoot":"","sources":["../../src/lib/dataServer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAKhE,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,WAAW,EACX,uBAAuB,EACvB,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,OAAO,EACP,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,oBAAoB,CAAC;AAG5B;;;;GAIG;AACH,QAAA,MAAM,UAAU;IACd;;;;;;OAMG;oBACa,qBAAqB,KAAG,gBAAgB;CA4CzD,CAAC;AACF,OAAO,EACL,UAAU,EACV,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,EACL,WAAW,EACX,uBAAuB,EACvB,kBAAkB,EAClB,SAAS,EACT,WAAW,EACX,UAAU,EACV,WAAW,GACZ,CAAC"}

package/dist/lib/dataServer.js

@@ -14,6 +14,7 @@ const query_types_1 = require("./query/query.types");
 Object.defineProperty(exports, "CacheStatus", { enumerable: true, get: function () { return query_types_1.CacheStatus; } });
 const dataServer_types_1 = require("./dataServer.types");
 Object.defineProperty(exports, "RunMode", { enumerable: true, get: function () { return dataServer_types_1.RunMode; } });
+const dumpIndexHelper_1 = require("./store/dump/dumpIndexHelper");
 /**
  * The Data Server instance.
  *
@@ -28,6 +29,7 @@ const dataServer = {
      * @returns An object with the data store and the queryRunner service.
      */
     init: (options) => {
+        const startDate = Date.now();
         // Configure logger.
         (0, logger_1.configureLogger)(options.logLevel, options.logFile);
         // Configure data server.
@@ -46,6 +48,15 @@ const dataServer = {
         }
         // Load data from dataDir.
         dataDir_1.dataDirManager.load();
+        // Create and load dump index.
+        if (config.dumpDir) {
+            if (dumpIndexHelper_1.dumpIndexHelper.isDumpIndexStale() ||
+                !dumpIndexHelper_1.dumpIndexHelper.isDumpIndexPresent()) {
+                dumpIndexHelper_1.dumpIndexHelper.createDumpIndex();
+            }
+            dumpIndexHelper_1.dumpIndexHelper.loadDumpIndex();
+        }
+        logger_1.logger.info(`Data Server loaded in ${Date.now() - startDate}ms.`);
         return {
             store: store_1.store,
             queryRunner: query_1.queryRunner,

package/dist/lib/store/diff/diff.types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"diff.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diff.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACnD,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAErB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACtB,CAAC"}
+{"version":3,"file":"diff.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diff.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACnD,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAErB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACtB,CAAC"}

package/dist/lib/store/diff/diffManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"diffManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diffManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAQ,MAAM,cAAc,CAAC;AAQjD,eAAO,MAAM,WAAW,EAAE,WAuEzB,CAAC"}
+{"version":3,"file":"diffManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diffManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAQ,MAAM,cAAc,CAAC;AAQjD,eAAO,MAAM,WAAW,EAAE,WAwEzB,CAAC"}

package/dist/lib/store/dump/dump.types.d.ts

@@ -72,4 +72,8 @@ export type DumpMetadata = {
     current: string;
     dumps: Dump[];
 };
+export type IndexEntry = {
+    hash: string;
+    url: string | null;
+};
 //# sourceMappingURL=dump.types.d.ts.map

package/dist/lib/store/dump/dump.types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dump.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dump.types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE1C;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAE/C;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,OAAO,EAAE,GAAG,CACV,MAAM,EACN;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAC7D,CAAC;IAEF;;;;;OAKG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,IAAI,CAAA;KAAE,CAAC,CAAC;IAE1E;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf,CAAC"}
+{"version":3,"file":"dump.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dump.types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE1C;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAE/C;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,OAAO,EAAE,GAAG,CACV,MAAM,EACN;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAC7D,CAAC;IAEF;;;;;OAKG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,IAAI,CAAA;KAAE,CAAC,CAAC;IAE1E;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB,CAAC"}

package/dist/lib/store/dump/dumpIndexHelper.d.ts

@@ -0,0 +1,14 @@
+import { IndexEntry } from './dump.types';
+export declare const dumpIndexHelper: {
+    isDumpIndexStale: () => boolean;
+    isDumpIndexPresent: () => boolean;
+    saveDumpIndex: () => void;
+    createDumpIndex: () => void;
+    loadDumpIndex: () => void;
+    hasEntry: (relativeFilePath: string) => boolean;
+    getEntry: (relativeFilePath: string) => IndexEntry | undefined;
+    getKeys: () => MapIterator<string>;
+    addEntry: (relativeFilePath: string, entry: IndexEntry) => void;
+    removeEntry: (relativeFilePath: string) => void;
+};
+//# sourceMappingURL=dumpIndexHelper.d.ts.map

package/dist/lib/store/dump/dumpIndexHelper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dumpIndexHelper.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpIndexHelper.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAiC1C,eAAO,MAAM,eAAe;4BACJ,OAAO;8BAYL,OAAO;yBAEZ,IAAI;2BASF,IAAI;yBAgCN,IAAI;iCAqBM,MAAM,KAAG,OAAO;iCAEhB,MAAM,KAAG,UAAU,GAAG,SAAS;;iCAK/B,MAAM,SAAS,UAAU,KAAG,IAAI;oCAK7B,MAAM,KAAG,IAAI;CAI9C,CAAC"}

package/dist/lib/store/dump/dumpIndexHelper.js

@@ -0,0 +1,112 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dumpIndexHelper = void 0;
+const fs_1 = __importDefault(require("fs"));
+const logger_1 = require("../../utils/logger");
+const string_1 = require("../../utils/string");
+const config_1 = require("../../config");
+// import { Dump, DumpMetadata } from './dump.types';
+// import { unixEpochUniqueId } from '../workDir';
+const fs_2 = require("../../utils/fs");
+const index = new Map();
+let isStale = undefined;
+const getIndexFilepath = () => `${config_1.config.dumpDir}/files.idx`;
+const getIndexIsStaleFlagFilepath = () => `${config_1.config.dumpDir}/index-is-stale.flag`;
+const indexToString = () => {
+    const indexLines = [];
+    index.forEach((value, filePath) => {
+        const line = [filePath, value.hash, value.url];
+        indexLines.push(line.join('\t'));
+    });
+    return indexLines.join('\n');
+};
+const setIsStale = (value) => {
+    if (isStale !== value) {
+        isStale = value;
+        const indexIsStaleFlagFilepath = getIndexIsStaleFlagFilepath();
+        if (isStale === true) {
+            fs_1.default.writeFileSync(indexIsStaleFlagFilepath, '');
+        }
+        else {
+            if (fs_1.default.existsSync(indexIsStaleFlagFilepath)) {
+                fs_1.default.unlinkSync(indexIsStaleFlagFilepath);
+            }
+        }
+    }
+};
+exports.dumpIndexHelper = {
+    isDumpIndexStale: () => {
+        if (isStale === undefined) {
+            const indexIsStaleFlagFilepath = getIndexIsStaleFlagFilepath();
+            if (fs_1.default.existsSync(indexIsStaleFlagFilepath)) {
+                setIsStale(true);
+            }
+            else {
+                setIsStale(false);
+            }
+        }
+        return !!isStale;
+    },
+    isDumpIndexPresent: () => fs_1.default.existsSync(getIndexFilepath()),
+    saveDumpIndex: () => {
+        if (isStale) {
+            const indexFilepath = getIndexFilepath();
+            const indexAsString = indexToString().trim();
+            fs_1.default.writeFileSync(indexFilepath, indexAsString);
+            setIsStale(false);
+        }
+    },
+    createDumpIndex: () => {
+        if (config_1.config.dumpDir) {
+            setIsStale(true);
+            logger_1.logger.info('Creating dump index...');
+            const startDate = Date.now();
+            index.clear();
+            const filesDumpDir = `${config_1.config.dumpDir}/files`;
+            const relativeFilePaths = (0, fs_2.findFilesInDir)(filesDumpDir);
+            relativeFilePaths.forEach(relativeFilePath => {
+                const fileContent = (0, fs_2.getFileContent)(`${filesDumpDir}/${relativeFilePath}`);
+                if (fileContent.raw) {
+                    const url = fileContent.json
+                        ? fileContent.json.data?.content?.url?.path
+                        : null;
+                    const hash = (0, string_1.createHash)(fileContent.raw);
+                    index.set(relativeFilePath, { hash, url });
+                }
+            });
+            // Save index into disk
+            exports.dumpIndexHelper.saveDumpIndex();
+            logger_1.logger.info(`${relativeFilePaths.length} files in dump indexed in ${Date.now() - startDate}ms.`);
+        }
+    },
+    loadDumpIndex: () => {
+        if (config_1.config.dumpDir) {
+            logger_1.logger.info('Loading dump index...');
+            const startDate = Date.now();
+            const indexFilepath = getIndexFilepath();
+            const data = fs_1.default.readFileSync(indexFilepath).toString().trim();
+            index.clear();
+            // Split lines and remove empty ones.
+            const dataLines = data.split('\n').filter(line => !!line);
+            dataLines.forEach(line => {
+                const [relativeFilePath, hash, url] = line.split('\t');
+                index.set(relativeFilePath, { hash, url });
+            });
+            logger_1.logger.info(`${dataLines.length} entries loaded from dump index in ${Date.now() - startDate}ms.`);
+        }
+    },
+    hasEntry: (relativeFilePath) => index.has(relativeFilePath),
+    getEntry: (relativeFilePath) => index.get(relativeFilePath),
+    getKeys: () => index.keys(),
+    addEntry: (relativeFilePath, entry) => {
+        setIsStale(true);
+        index.set(relativeFilePath, entry);
+    },
+    removeEntry: (relativeFilePath) => {
+        setIsStale(true);
+        index.delete(relativeFilePath);
+    },
+};

package/dist/lib/store/dump/dumpManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dumpManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpManager.ts"],"names":[],"mappings":"AAcA,OAAO,EAAQ,WAAW,EAAE,MAAM,cAAc,CAAC;AA2GjD,eAAO,MAAM,WAAW,EAAE,WA8EzB,CAAC"}
+{"version":3,"file":"dumpManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpManager.ts"],"names":[],"mappings":"AASA,OAAO,EAAQ,WAAW,EAAE,MAAM,cAAc,CAAC;AAqKjD,eAAO,MAAM,WAAW,EAAE,WA+EzB,CAAC"}

package/dist/lib/store/dump/dumpManager.js

@@ -14,59 +14,49 @@ const store_1 = require("../store");
 const diffManager_1 = require("../diff/diffManager");
 const hook_1 = require("../hook");
 const dumpMetadataHelper_1 = require("./dumpMetadataHelper");
-const storeUpdatedFiles = (diff, dump, dumpDir) => {
+const dumpIndexHelper_1 = require("./dumpIndexHelper");
+const string_1 = require("../../utils/string");
+const storeUpdatedFiles = (diff, dump, filesDumpDir) => {
     diff.updated.forEach(relativeFilepath => {
         const storeFileContentData = store_1.store.data.get(relativeFilepath);
         if (storeFileContentData) {
-            const absoluteFilepathInDumpDir = `${dumpDir}/${relativeFilepath}`;
+            const absoluteFilepathInDumpDir = `${filesDumpDir}/${relativeFilepath}`;
             try {
-                // Create parent directories if they are missing.
-                const dir = path_1.default.dirname(absoluteFilepathInDumpDir);
-                if (!fs_1.default.existsSync(dir)) {
-                    fs_1.default.mkdirSync(dir, { recursive: true });
-                }
                 // If store data is an object, stringify it to execute its
                 // queries and resolve its includes.
                 const isJson = (0, fs_2.isJsonFile)(relativeFilepath);
                 const storeFileContentString = isJson
                     ? JSON.stringify(storeFileContentData)
                     : storeFileContentData;
-                // Before overwriting a file, check it has changed.
+                // Before overwriting a file, check if it has changed.
                 let needsSave = true;
                 let oldPublicUrl = null;
                 const newPublicUrl = storeFileContentData.data?.content?.url?.path || null;
-                if (fs_1.default.existsSync(absoluteFilepathInDumpDir)) {
-                    const dumpFileContentString = (0, fs_2.readFile)(absoluteFilepathInDumpDir);
-                    if (storeFileContentString === dumpFileContentString) {
+                const newHash = (0, string_1.createHash)(storeFileContentString);
+                const indexEntry = dumpIndexHelper_1.dumpIndexHelper.getEntry(relativeFilepath);
+                if (indexEntry) {
+                    if (newHash === indexEntry.hash) {
                         // No need to save it, file has not changed.
                         needsSave = false;
                     }
-                    else if (isJson && dumpFileContentString) {
-                        // Get old public URL before it changes on disk.
-                        oldPublicUrl =
-                            JSON.parse(dumpFileContentString)?.data?.content?.url?.path ||
-                                null;
-                        /*
-                        // Save a copy for later reference.
-                        const backupAbsoluteFilepathInDumpDir =
-                          absoluteFilepathInDumpDir.replace(
-                            '/dump/files/',
-                            '/dump/files.bak/',
-                          );
-                        const backupDir = path.dirname(backupAbsoluteFilepathInDumpDir);
-                        if (!fs.existsSync(backupDir)) {
-                          fs.mkdirSync(backupDir, { recursive: true });
-                        }
-                        fs.renameSync(
-                          absoluteFilepathInDumpDir,
-                          backupAbsoluteFilepathInDumpDir,
-                        );
-                        */
+                    else if (isJson) {
+                        // Get old public URL before it changes on disk and index.
+                        oldPublicUrl = indexEntry.url;
                     }
                 }
                 if (needsSave) {
-                    // Save it.
+                    // Create parent directories if they are missing.
+                    const dir = path_1.default.dirname(absoluteFilepathInDumpDir);
+                    if (!fs_1.default.existsSync(dir)) {
+                        fs_1.default.mkdirSync(dir, { recursive: true });
+                    }
+                    // Save it on disk.
                     fs_1.default.writeFileSync(absoluteFilepathInDumpDir, storeFileContentString);
+                    // Save it to index.
+                    dumpIndexHelper_1.dumpIndexHelper.addEntry(relativeFilepath, {
+                        hash: newHash,
+                        url: newPublicUrl,
+                    });
                     // Mark it as updated.
                     dump.updated.set(relativeFilepath, {
                         oldPublicUrl,
@@ -83,18 +73,20 @@ const storeUpdatedFiles = (diff, dump, dumpDir) => {
         }
     });
 };
-const removeDeletedFiles = (diff, dump, dumpDir) => {
+const removeDeletedFiles = (diff, dump, filesDumpDir) => {
     diff.deleted.forEach(relativeFilepath => {
-        const absoluteFilepath = `${dumpDir}/${relativeFilepath}`;
+        const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`;
         try {
-            if (fs_1.default.existsSync(absoluteFilepath)) {
-                // Get old public URLs before they change on disk.
-                const dumpContent = (0, fs_2.getFileContent)(absoluteFilepath);
-                const oldPublicUrl = dumpContent.json?.data?.content?.url?.path;
+            const indexEntry = dumpIndexHelper_1.dumpIndexHelper.getEntry(relativeFilepath);
+            if (indexEntry) {
+                // Get old public URLs before they change on disk and index.
+                const oldPublicUrl = indexEntry.url;
                 // Delete it.
                 fs_1.default.unlinkSync(absoluteFilepath);
                 // Delete any empty directory.
                 (0, fs_2.removeEmptyDirsUpwards)(path_1.default.dirname(absoluteFilepath));
+                // Delete it from index.
+                dumpIndexHelper_1.dumpIndexHelper.removeEntry(relativeFilepath);
                 // Mark it as deleted.
                 dump.deleted.set(relativeFilepath, {
                     oldPublicUrl,
@@ -107,6 +99,48 @@ const removeDeletedFiles = (diff, dump, dumpDir) => {
         }
     });
 };
+const removeStaleFiles = (diff, dump, filesDumpDir) => {
+    const iterator = dumpIndexHelper_1.dumpIndexHelper.getKeys();
+    for (const relativeFilepath of iterator) {
+        const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`;
+        if (!store_1.store.data.has(relativeFilepath) && fs_1.default.existsSync(absoluteFilepath)) {
+            logger_1.logger.info(`Dump: removing stale file: ${relativeFilepath}`);
+            try {
+                // Get old public URLs before they change on disk.
+                const dumpStaleFileString = fs_1.default
+                    .readFileSync(absoluteFilepath)
+                    .toString();
+                const isJson = (0, fs_2.isJsonFile)(relativeFilepath);
+                const dumpStaleFileContentData = isJson
+                    ? JSON.parse(dumpStaleFileString)
+                    : dumpStaleFileString;
+                let oldPublicUrl = dumpStaleFileContentData.data?.content?.url?.path || null;
+                // An stale file can be a manually copied file.
+                // In that case, that stale file would have an oldPublicUrl from a valid file,
+                // which would lead to deleting a valid URL.
+                // To solve that case, check that oldPublicUrl is not a valid URL.
+                if (oldPublicUrl && store_1.store.index.url.has(oldPublicUrl)) {
+                    logger_1.logger.info(`Dump: skipping valid url from stale file: ${oldPublicUrl}`);
+                    oldPublicUrl = null;
+                }
+                // Delete it.
+                fs_1.default.unlinkSync(absoluteFilepath);
+                // Delete any empty directory.
+                (0, fs_2.removeEmptyDirsUpwards)(path_1.default.dirname(absoluteFilepath));
+                // Delete it from index.
+                dumpIndexHelper_1.dumpIndexHelper.removeEntry(relativeFilepath);
+                // Mark it as deleted.
+                dump.deleted.set(relativeFilepath, {
+                    oldPublicUrl,
+                    newPublicUrl: null,
+                });
+            }
+            catch (e) {
+                logger_1.logger.error(`Dump: error deleting stale file "${absoluteFilepath}": ${e}`);
+            }
+        }
+    }
+};
 exports.dumpManager = {
     dump(options = { incremental: true }) {
         const startDate = microtime_1.default.now();
@@ -121,12 +155,17 @@ exports.dumpManager = {
             diff,
         };
         if (config_1.config.dumpDir) {
-            const dumpDir = `${config_1.config.dumpDir}/files`;
+            const filesDumpDir = `${config_1.config.dumpDir}/files`;
             if (diff.updated.size || diff.deleted.size) {
+                dumpMetadataHelper_1.dumpMetadataHelper.storeDumpMetadata(dump);
                 // Store updated files.
-                storeUpdatedFiles(diff, dump, dumpDir);
+                storeUpdatedFiles(diff, dump, filesDumpDir);
                 // Remove deleted files.
-                removeDeletedFiles(diff, dump, dumpDir);
+                removeDeletedFiles(diff, dump, filesDumpDir);
+                // Remove stale files.
+                removeStaleFiles(diff, dump, filesDumpDir);
+                // Save dump index
+                dumpIndexHelper_1.dumpIndexHelper.saveDumpIndex();
                 // Invoke "onDumpCreate" hook.
                 dump = hook_1.hookManager.invokeOnDumpCreate(dump);
                 const execTimeMs = (microtime_1.default.now() - startDate) / 1000;
@@ -140,12 +179,6 @@ exports.dumpManager = {
                         diffManager_1.diffManager.reset(diff.toUniqueId);
                     }
                     logger_1.logger.info(`Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`);
-                    /*
-                    // Log dump if not empty
-                    if (dump.updated.size || dump.deleted.size) {
-                      logger.debug(`Dump: "${JSON.stringify(jsonify(dump))}"`);
-                    }
-                    */
                 }
                 else {
                     // Resetting the diff must happen when no other operations are pending.

package/dist/lib/store/dump/dumpMetadataHelper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dumpMetadataHelper.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpMetadataHelper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAKlD,eAAO,MAAM,kBAAkB;8BACH,IAAI,KAAG,OAAO;sCA6BN,MAAM,GAAG,YAAY;8BAuB7B,MAAM;CAWjC,CAAC"}
+{"version":3,"file":"dumpMetadataHelper.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpMetadataHelper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAKlD,eAAO,MAAM,kBAAkB;8BACH,IAAI,KAAG,OAAO;sCA6BN,MAAM,GAAG,YAAY;8BAuB7B,MAAM;CAYjC,CAAC"}

package/dist/lib/store/dump/dumpMetadataHelper.js

@@ -54,6 +54,7 @@ exports.dumpMetadataHelper = {
     },
     getCurrentDumpUniqueId() {
         const metadataFilepath = getMetadataFilepath();
+        logger_1.logger.debug(`metadataFilepath: ${metadataFilepath}`);
         const metadata = JSON.parse(fs_1.default.existsSync(metadataFilepath)
             ? fs_1.default.readFileSync(metadataFilepath).toString()
             : '[]');

package/dist/lib/utils/string/stringUtils.d.ts

@@ -48,4 +48,10 @@ export declare const parseUniqueId: (uniqueId: string) => Date | null;
  * @param uniqueId - A unique id to be checked
  */
 export declare const isUniqueId: (uniqueId: string) => boolean;
+/**
+ * Creates a hash from a string
+ *
+ * @param data - A string of data
+ */
+export declare const createHash: (data: string) => string;
 //# sourceMappingURL=stringUtils.d.ts.map

package/dist/lib/utils/string/stringUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stringUtils.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/string/stringUtils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEvD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,MAAM,KAAG,GAQpD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,GAC/B,aAAa,MAAM,KAClB,qBAWF,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,GAAI,UAAU,MAAM,KAAG,IAAI,GAAG,IAevD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,UAAU,GAAI,UAAU,MAAM,KAAG,OAC+B,CAAC"}
+{"version":3,"file":"stringUtils.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/string/stringUtils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAEvD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,MAAM,KAAG,GAQpD,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,GAC/B,aAAa,MAAM,KAClB,qBAWF,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,GAAI,UAAU,MAAM,KAAG,IAAI,GAAG,IAevD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,UAAU,GAAI,UAAU,MAAM,KAAG,OAC+B,CAAC;AAE9E;;;;GAIG;AACH,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,MAC0B,CAAC"}

package/dist/lib/utils/string/stringUtils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isUniqueId = exports.parseUniqueId = exports.parseURLSearchParams = exports.parseJsonString = exports.VARIANT_SEPARATOR = void 0;
+exports.createHash = exports.isUniqueId = exports.parseUniqueId = exports.parseURLSearchParams = exports.parseJsonString = exports.VARIANT_SEPARATOR = void 0;
 const logger_1 = require("../logger");
 /**
  * Separator for variant data files saved to storage.
@@ -87,3 +87,10 @@ exports.parseUniqueId = parseUniqueId;
  */
 const isUniqueId = (uniqueId) => /(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/.test(uniqueId);
 exports.isUniqueId = isUniqueId;
+/**
+ * Creates a hash from a string
+ *
+ * @param data - A string of data
+ */
+const createHash = (data) => require('crypto').createHash('sha1').update(data).digest('base64');
+exports.createHash = createHash;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@static-suite/static-suite-data-server",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "A flexible, fast and developer friendly Data Server for Static Suite",
   "main": "./dist/lib/index.js",
   "typings": "./dist/lib/index.d.ts",
@@ -84,5 +84,6 @@
       "src/__tests__/fixtures/*",
       "src/__tests__/mocks/*"
     ]
-  }
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

package/src/http/routes/diff.ts

@@ -0,0 +1,47 @@
+import { Request, Response } from 'express';
+import { diffManager } from '../../lib/store/diff';
+import { jsonify } from '../../lib/utils/object';
+
+const diffIndex = (req: Request, res: Response): void => {
+  res.render('diffIndex', {
+    links: {
+      '/diff/incremental': 'Execute an incremental dump',
+      '/diff/full': 'Execute a full dump',
+      '/diff/reset':
+        'Clears intermediate changes tracked by dependency manager',
+    },
+  });
+};
+
+const diffAction = (
+  req: Request,
+  res: Response,
+  incremental: boolean,
+): void => {
+  const diff = diffManager.getDiff({ incremental });
+  const diffAsJson: any = jsonify(diff);
+
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send(diffAsJson);
+};
+
+const diffIncremental = (req: Request, res: Response): void => {
+  diffAction(req, res, true);
+};
+
+const diffFull = (req: Request, res: Response): void => {
+  diffAction(req, res, false);
+};
+
+const diffReset = (req: Request, res: Response): void => {
+  const args: any = req.query;
+  if (args?.uniqueId) {
+    diffManager.reset(args.uniqueId);
+  }
+  res.status(200);
+  res.set({ 'Content-Type': 'application/json' });
+  res.send({ status: 'done' });
+};
+
+export { diffIndex, diffIncremental, diffFull, diffReset };

package/src/http/routes/home.ts

@@ -5,6 +5,7 @@ export const home = (req: Request, res: Response): void => {
     links: {
       '/data': 'Browse all data loaded into the server',
       '/dependency': 'Dependency data',
+      '/diff': 'Diff operations',
       '/dump': 'Dump operations',
       '/query': 'List of available queries',
       '/task': 'List of available tasks',

package/src/http/routes/index.ts

@@ -32,6 +32,7 @@ import {
 import { indexUrl } from './indexUrl';
 import { indexUUID } from './indexUUID';
 import { subset } from './subset';
+import { diffFull, diffIncremental, diffIndex, diffReset } from './diff';
 
 const routes = Router();
 
@@ -49,6 +50,10 @@ routes.get('/task', taskIndex);
 routes.get('/task/:taskId', runTask);
 routes.get('/cache', cacheIndex);
 routes.get('/cache/:cacheBinId/clear', cacheClear);
+routes.get('/diff', diffIndex);
+routes.get('/diff/incremental', diffIncremental);
+routes.get('/diff/full', diffFull);
+routes.get('/diff/reset', diffReset);
 routes.get('/dump', dumpIndex);
 routes.get('/dump/incremental', dumpIncremental);
 routes.get('/dump/full', dumpFull);

package/src/http/routes/subset.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from 'express';
-import { StoreSubsetOptions } from 'lib/store/store.types';
+import { StoreSubsetOptions } from '../../lib/store/store.types';
 import { store } from '../../lib/store';
 import { dataDirManager } from '../../lib/store/dataDir';
 

package/src/http/views/diffIndex.pug

@@ -0,0 +1,14 @@
+extends includes/layout.pug
+
+block vars
+  - var title = "Diff | Static Suite Data Server"
+
+block content
+  h1(class="title is-1")="Diff"
+  table(class="table is-fullwidth is-hoverable")
+    tbody
+      each description, link in links
+        tr
+          td
+            a(href=link)= link
+          td=description

package/src/lib/dataServer.ts

@@ -1,4 +1,4 @@
-import { configureLogger } from './utils/logger';
+import { configureLogger, logger } from './utils/logger';
 import { LogLevel, LogFile } from './utils/logger/logger.types';
 import { initWatcher } from './watcher';
 import { setConfig } from './config';
@@ -21,6 +21,7 @@ import {
   DataServerReturn,
   DataServerInitOptions,
 } from './dataServer.types';
+import { dumpIndexHelper } from './store/dump/dumpIndexHelper';
 
 /**
  * The Data Server instance.
@@ -36,6 +37,8 @@ const dataServer = {
    * @returns An object with the data store and the queryRunner service.
    */
   init: (options: DataServerInitOptions): DataServerReturn => {
+    const startDate = Date.now();
+
     // Configure logger.
     configureLogger(options.logLevel, options.logFile);
 
@@ -58,6 +61,19 @@ const dataServer = {
     // Load data from dataDir.
     dataDirManager.load();
 
+    // Create and load dump index.
+    if (config.dumpDir) {
+      if (
+        dumpIndexHelper.isDumpIndexStale() ||
+        !dumpIndexHelper.isDumpIndexPresent()
+      ) {
+        dumpIndexHelper.createDumpIndex();
+      }
+      dumpIndexHelper.loadDumpIndex();
+    }
+
+    logger.info(`Data Server loaded in ${Date.now() - startDate}ms.`);
+
     return {
       store,
       queryRunner,

package/src/lib/store/diff/diff.types.ts

@@ -38,6 +38,7 @@ export type Diff = {
    * Execution time taken by the diff, in milliseconds.
    */
   execTimeMs: number;
+
   /**
    * A unique id representing the date from which this diff is obtained.
    */

package/src/lib/store/diff/diffManager.ts

@@ -27,6 +27,7 @@ export const diffManager: DiffManager = {
     const currentDumpUniqueId = dumpMetadataHelper.getCurrentDumpUniqueId();
     const isFirstDiffAfterReboot = lastDiffUniqueId === unixEpochUniqueId;
     const isADumpAvailable = currentDumpUniqueId !== unixEpochUniqueId;
+
     // If both checks passes, check if something has changed after last dump to avoid a full diff:
     if (isFirstDiffAfterReboot && isADumpAvailable) {
       const dataDirModificationUniqueId = changedFiles.toUniqueId;

package/src/lib/store/dump/dump.types.ts

@@ -76,3 +76,8 @@ export type DumpMetadata = {
   current: string;
   dumps: Dump[];
 };
+
+export type IndexEntry = {
+  hash: string;
+  url: string | null;
+};

package/src/lib/store/dump/dumpIndexHelper.ts

@@ -0,0 +1,134 @@
+import fs from 'fs';
+import { logger } from '../../utils/logger';
+import { createHash } from '../../utils/string';
+import { config } from '../../config';
+// import { Dump, DumpMetadata } from './dump.types';
+// import { unixEpochUniqueId } from '../workDir';
+import { findFilesInDir, getFileContent } from '../../utils/fs';
+import { IndexEntry } from './dump.types';
+
+const index = new Map<string, IndexEntry>();
+
+let isStale: boolean | undefined = undefined;
+
+const getIndexFilepath = () => `${config.dumpDir}/files.idx`;
+const getIndexIsStaleFlagFilepath = () =>
+  `${config.dumpDir}/index-is-stale.flag`;
+
+const indexToString = (): string => {
+  const indexLines: string[] = [];
+  index.forEach((value, filePath) => {
+    const line = [filePath, value.hash, value.url];
+    indexLines.push(line.join('\t'));
+  });
+  return indexLines.join('\n');
+};
+
+const setIsStale = (value: boolean): void => {
+  if (isStale !== value) {
+    isStale = value;
+    const indexIsStaleFlagFilepath = getIndexIsStaleFlagFilepath();
+    if (isStale === true) {
+      fs.writeFileSync(indexIsStaleFlagFilepath, '');
+    } else {
+      if (fs.existsSync(indexIsStaleFlagFilepath)) {
+        fs.unlinkSync(indexIsStaleFlagFilepath);
+      }
+    }
+  }
+};
+
+export const dumpIndexHelper = {
+  isDumpIndexStale: (): boolean => {
+    if (isStale === undefined) {
+      const indexIsStaleFlagFilepath = getIndexIsStaleFlagFilepath();
+      if (fs.existsSync(indexIsStaleFlagFilepath)) {
+        setIsStale(true);
+      } else {
+        setIsStale(false);
+      }
+    }
+    return !!isStale;
+  },
+
+  isDumpIndexPresent: (): boolean => fs.existsSync(getIndexFilepath()),
+
+  saveDumpIndex: (): void => {
+    if (isStale) {
+      const indexFilepath = getIndexFilepath();
+      const indexAsString = indexToString().trim();
+      fs.writeFileSync(indexFilepath, indexAsString);
+      setIsStale(false);
+    }
+  },
+
+  createDumpIndex: (): void => {
+    if (config.dumpDir) {
+      setIsStale(true);
+      logger.info('Creating dump index...');
+      const startDate = Date.now();
+      index.clear();
+      const filesDumpDir = `${config.dumpDir}/files`;
+      const relativeFilePaths = findFilesInDir(filesDumpDir);
+      relativeFilePaths.forEach(relativeFilePath => {
+        const fileContent = getFileContent(
+          `${filesDumpDir}/${relativeFilePath}`,
+        );
+        if (fileContent.raw) {
+          const url = fileContent.json
+            ? fileContent.json.data?.content?.url?.path
+            : null;
+          const hash = createHash(fileContent.raw);
+          index.set(relativeFilePath, { hash, url });
+        }
+      });
+
+      // Save index into disk
+      dumpIndexHelper.saveDumpIndex();
+
+      logger.info(
+        `${relativeFilePaths.length} files in dump indexed in ${
+          Date.now() - startDate
+        }ms.`,
+      );
+    }
+  },
+
+  loadDumpIndex: (): void => {
+    if (config.dumpDir) {
+      logger.info('Loading dump index...');
+      const startDate = Date.now();
+      const indexFilepath = getIndexFilepath();
+      const data = fs.readFileSync(indexFilepath).toString().trim();
+      index.clear();
+      // Split lines and remove empty ones.
+      const dataLines: string[] = data.split('\n').filter(line => !!line);
+      dataLines.forEach(line => {
+        const [relativeFilePath, hash, url] = line.split('\t');
+        index.set(relativeFilePath, { hash, url });
+      });
+      logger.info(
+        `${dataLines.length} entries loaded from dump index in ${
+          Date.now() - startDate
+        }ms.`,
+      );
+    }
+  },
+
+  hasEntry: (relativeFilePath: string): boolean => index.has(relativeFilePath),
+
+  getEntry: (relativeFilePath: string): IndexEntry | undefined =>
+    index.get(relativeFilePath),
+
+  getKeys: () => index.keys(),
+
+  addEntry: (relativeFilePath: string, entry: IndexEntry): void => {
+    setIsStale(true);
+    index.set(relativeFilePath, entry);
+  },
+
+  removeEntry: (relativeFilePath: string): void => {
+    setIsStale(true);
+    index.delete(relativeFilePath);
+  },
+};

package/src/lib/store/dump/dumpManager.ts

@@ -2,12 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import microtime from 'microtime';
 import { logger } from '../../utils/logger';
-import {
-  getFileContent,
-  isJsonFile,
-  readFile,
-  removeEmptyDirsUpwards,
-} from '../../utils/fs';
+import { isJsonFile, removeEmptyDirsUpwards } from '../../utils/fs';
 import { config } from '../../config';
 import { store } from '../store';
 import { diffManager } from '../diff/diffManager';
@@ -15,19 +10,19 @@ import { Diff } from '../diff/diff.types';
 import { Dump, DumpManager } from './dump.types';
 import { hookManager } from '../hook';
 import { dumpMetadataHelper } from './dumpMetadataHelper';
-
-const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
+import { dumpIndexHelper } from './dumpIndexHelper';
+import { createHash } from '../../utils/string';
+
+const storeUpdatedFiles = (
+  diff: Diff,
+  dump: Dump,
+  filesDumpDir: string,
+): void => {
   diff.updated.forEach(relativeFilepath => {
     const storeFileContentData = store.data.get(relativeFilepath);
     if (storeFileContentData) {
-      const absoluteFilepathInDumpDir = `${dumpDir}/${relativeFilepath}`;
+      const absoluteFilepathInDumpDir = `${filesDumpDir}/${relativeFilepath}`;
       try {
-        // Create parent directories if they are missing.
-        const dir = path.dirname(absoluteFilepathInDumpDir);
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir, { recursive: true });
-        }
-
         // If store data is an object, stringify it to execute its
         // queries and resolve its includes.
         const isJson = isJsonFile(relativeFilepath);
@@ -35,43 +30,39 @@ const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
           ? JSON.stringify(storeFileContentData)
           : storeFileContentData;
 
-        // Before overwriting a file, check it has changed.
+        // Before overwriting a file, check if it has changed.
         let needsSave = true;
         let oldPublicUrl: string | null = null;
         const newPublicUrl: string | null =
           storeFileContentData.data?.content?.url?.path || null;
-        if (fs.existsSync(absoluteFilepathInDumpDir)) {
-          const dumpFileContentString = readFile(absoluteFilepathInDumpDir);
-          if (storeFileContentString === dumpFileContentString) {
+        const newHash = createHash(storeFileContentString);
+        const indexEntry = dumpIndexHelper.getEntry(relativeFilepath);
+        if (indexEntry) {
+          if (newHash === indexEntry.hash) {
             // No need to save it, file has not changed.
             needsSave = false;
-          } else if (isJson && dumpFileContentString) {
-            // Get old public URL before it changes on disk.
-            oldPublicUrl =
-              JSON.parse(dumpFileContentString)?.data?.content?.url?.path ||
-              null;
-            /*
-            // Save a copy for later reference.
-            const backupAbsoluteFilepathInDumpDir =
-              absoluteFilepathInDumpDir.replace(
-                '/dump/files/',
-                '/dump/files.bak/',
-              );
-            const backupDir = path.dirname(backupAbsoluteFilepathInDumpDir);
-            if (!fs.existsSync(backupDir)) {
-              fs.mkdirSync(backupDir, { recursive: true });
-            }
-            fs.renameSync(
-              absoluteFilepathInDumpDir,
-              backupAbsoluteFilepathInDumpDir,
-            );
-            */
+          } else if (isJson) {
+            // Get old public URL before it changes on disk and index.
+            oldPublicUrl = indexEntry.url;
           }
         }
+
         if (needsSave) {
-          // Save it.
+          // Create parent directories if they are missing.
+          const dir = path.dirname(absoluteFilepathInDumpDir);
+          if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+          }
+
+          // Save it on disk.
           fs.writeFileSync(absoluteFilepathInDumpDir, storeFileContentString);
 
+          // Save it to index.
+          dumpIndexHelper.addEntry(relativeFilepath, {
+            hash: newHash,
+            url: newPublicUrl,
+          });
+
           // Mark it as updated.
           dump.updated.set(relativeFilepath, {
             oldPublicUrl,
@@ -91,15 +82,18 @@ const storeUpdatedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
   });
 };
 
-const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
+const removeDeletedFiles = (
+  diff: Diff,
+  dump: Dump,
+  filesDumpDir: string,
+): void => {
   diff.deleted.forEach(relativeFilepath => {
-    const absoluteFilepath = `${dumpDir}/${relativeFilepath}`;
+    const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`;
     try {
-      if (fs.existsSync(absoluteFilepath)) {
-        // Get old public URLs before they change on disk.
-        const dumpContent = getFileContent(absoluteFilepath);
-        const oldPublicUrl: string | null =
-          dumpContent.json?.data?.content?.url?.path;
+      const indexEntry = dumpIndexHelper.getEntry(relativeFilepath);
+      if (indexEntry) {
+        // Get old public URLs before they change on disk and index.
+        const oldPublicUrl = indexEntry.url;
 
         // Delete it.
         fs.unlinkSync(absoluteFilepath);
@@ -107,6 +101,9 @@ const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
         // Delete any empty directory.
         removeEmptyDirsUpwards(path.dirname(absoluteFilepath));
 
+        // Delete it from index.
+        dumpIndexHelper.removeEntry(relativeFilepath);
+
         // Mark it as deleted.
         dump.deleted.set(relativeFilepath, {
           oldPublicUrl,
@@ -119,6 +116,62 @@ const removeDeletedFiles = (diff: Diff, dump: Dump, dumpDir: string): void => {
   });
 };
 
+const removeStaleFiles = (
+  diff: Diff,
+  dump: Dump,
+  filesDumpDir: string,
+): void => {
+  const iterator = dumpIndexHelper.getKeys();
+  for (const relativeFilepath of iterator) {
+    const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`;
+    if (!store.data.has(relativeFilepath) && fs.existsSync(absoluteFilepath)) {
+      logger.info(`Dump: removing stale file: ${relativeFilepath}`);
+      try {
+        // Get old public URLs before they change on disk.
+        const dumpStaleFileString = fs
+          .readFileSync(absoluteFilepath)
+          .toString();
+        const isJson = isJsonFile(relativeFilepath);
+        const dumpStaleFileContentData = isJson
+          ? JSON.parse(dumpStaleFileString)
+          : dumpStaleFileString;
+        let oldPublicUrl: string | null =
+          dumpStaleFileContentData.data?.content?.url?.path || null;
+
+        // An stale file can be a manually copied file.
+        // In that case, that stale file would have an oldPublicUrl from a valid file,
+        // which would lead to deleting a valid URL.
+        // To solve that case, check that oldPublicUrl is not a valid URL.
+        if (oldPublicUrl && store.index.url.has(oldPublicUrl)) {
+          logger.info(
+            `Dump: skipping valid url from stale file: ${oldPublicUrl}`,
+          );
+          oldPublicUrl = null;
+        }
+
+        // Delete it.
+        fs.unlinkSync(absoluteFilepath);
+
+        // Delete any empty directory.
+        removeEmptyDirsUpwards(path.dirname(absoluteFilepath));
+
+        // Delete it from index.
+        dumpIndexHelper.removeEntry(relativeFilepath);
+
+        // Mark it as deleted.
+        dump.deleted.set(relativeFilepath, {
+          oldPublicUrl,
+          newPublicUrl: null,
+        });
+      } catch (e) {
+        logger.error(
+          `Dump: error deleting stale file "${absoluteFilepath}": ${e}`,
+        );
+      }
+    }
+  }
+};
+
 export const dumpManager: DumpManager = {
   dump(options = { incremental: true }): Dump {
     const startDate = microtime.now();
@@ -135,14 +188,22 @@ export const dumpManager: DumpManager = {
     };
 
     if (config.dumpDir) {
-      const dumpDir = `${config.dumpDir}/files`;
+      const filesDumpDir = `${config.dumpDir}/files`;
 
       if (diff.updated.size || diff.deleted.size) {
+        dumpMetadataHelper.storeDumpMetadata(dump);
+
         // Store updated files.
-        storeUpdatedFiles(diff, dump, dumpDir);
+        storeUpdatedFiles(diff, dump, filesDumpDir);
 
         // Remove deleted files.
-        removeDeletedFiles(diff, dump, dumpDir);
+        removeDeletedFiles(diff, dump, filesDumpDir);
+
+        // Remove stale files.
+        removeStaleFiles(diff, dump, filesDumpDir);
+
+        // Save dump index
+        dumpIndexHelper.saveDumpIndex();
 
         // Invoke "onDumpCreate" hook.
         dump = hookManager.invokeOnDumpCreate(dump);
@@ -162,13 +223,6 @@ export const dumpManager: DumpManager = {
           logger.info(
             `Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`,
           );
-
-          /*
-          // Log dump if not empty
-          if (dump.updated.size || dump.deleted.size) {
-            logger.debug(`Dump: "${JSON.stringify(jsonify(dump))}"`);
-          }
-          */
         } else {
           // Resetting the diff must happen when no other operations are pending.
           diffManager.reset(diff.toUniqueId);

package/src/lib/store/dump/dumpMetadataHelper.ts

@@ -63,6 +63,7 @@ export const dumpMetadataHelper = {
   getCurrentDumpUniqueId(): string {
     const metadataFilepath = getMetadataFilepath();
 
+    logger.debug(`metadataFilepath: ${metadataFilepath}`);
     const metadata = JSON.parse(
       fs.existsSync(metadataFilepath)
         ? fs.readFileSync(metadataFilepath).toString()

package/src/lib/utils/string/stringUtils.ts

@@ -91,3 +91,11 @@ export const parseUniqueId = (uniqueId: string): Date | null => {
  */
 export const isUniqueId = (uniqueId: string): boolean =>
   /(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/.test(uniqueId);
+
+/**
+ * Creates a hash from a string
+ *
+ * @param data - A string of data
+ */
+export const createHash = (data: string): string =>
+  require('crypto').createHash('sha1').update(data).digest('base64');

package/tsconfig.json

@@ -30,9 +30,9 @@
 
     /* Strict Type-Checking Options */
     "strict": true /* Enable all strict type-checking options. */,
-    "paths": {
+    /*  "paths": {
       "*": ["./src/*"]
-    } /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */,
+    } */ /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
     // "rootDirs": [],                              /* List of root folders whose combined content represents the structure of the project at runtime. */
     // "typeRoots": [],                             /* List of folders to include type definitions from. */
     "types": [