@static-suite/static-suite-data-server 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +569 -107
- package/dist/http/README.md +569 -107
- package/dist/http/routes/dependency.d.ts +10 -0
- package/dist/http/routes/dependency.d.ts.map +1 -0
- package/dist/http/routes/dependency.js +71 -0
- package/dist/http/routes/dump.d.ts.map +1 -1
- package/dist/http/routes/dump.js +15 -10
- package/dist/http/routes/home.js +2 -1
- package/dist/http/routes/index.d.ts.map +1 -1
- package/dist/http/routes/index.js +8 -0
- package/dist/http/routes/status.d.ts.map +1 -1
- package/dist/http/routes/status.js +1 -6
- package/dist/http/views/dependencyIndex.pug +14 -0
- package/dist/lib/dataServer.d.ts +2 -1
- package/dist/lib/dataServer.d.ts.map +1 -1
- package/dist/lib/query/query.types.d.ts +2 -2
- package/dist/lib/query/query.types.d.ts.map +1 -1
- package/dist/lib/query/query.types.js +2 -2
- package/dist/lib/query/queryRunner.d.ts.map +1 -1
- package/dist/lib/query/queryRunner.js +3 -1
- package/dist/lib/store/dataDir/dataDir.types.d.ts +5 -5
- package/dist/lib/store/dataDir/dataDir.types.d.ts.map +1 -1
- package/dist/lib/store/dataDir/dataDirManager.d.ts.map +1 -1
- package/dist/lib/store/dataDir/dataDirManager.js +23 -12
- package/dist/lib/store/dependency/dependency.types.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.d.ts +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.js +5 -3
- package/dist/lib/store/dependency/dependencyManager.d.ts +12 -0
- package/dist/lib/store/dependency/dependencyManager.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyManager.js +7 -5
- package/dist/lib/store/dependency/dependencyTagger.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyTagger.js +3 -38
- package/dist/lib/store/diff/{diffManager.types.d.ts → diff.types.d.ts} +14 -6
- package/dist/lib/store/diff/diff.types.d.ts.map +1 -0
- package/dist/lib/store/{dependency/dependencyManager.types.js → diff/diff.types.js} +0 -0
- package/dist/lib/store/diff/diffManager.d.ts +1 -1
- package/dist/lib/store/diff/diffManager.d.ts.map +1 -1
- package/dist/lib/store/diff/diffManager.js +38 -81
- package/dist/lib/store/dump/{dumpManager.types.d.ts → dump.types.d.ts} +21 -8
- package/dist/lib/store/dump/dump.types.d.ts.map +1 -0
- package/dist/lib/store/{diff/diffManager.types.js → dump/dump.types.js} +0 -0
- package/dist/lib/store/dump/dumpManager.d.ts +1 -1
- package/dist/lib/store/dump/dumpManager.d.ts.map +1 -1
- package/dist/lib/store/dump/dumpManager.js +31 -56
- package/dist/lib/store/dump/dumpMetadataHelper.d.ts +7 -0
- package/dist/lib/store/dump/dumpMetadataHelper.d.ts.map +1 -0
- package/dist/lib/store/dump/dumpMetadataHelper.js +61 -0
- package/dist/lib/store/hook/hook.types.d.ts +49 -23
- package/dist/lib/store/hook/hook.types.d.ts.map +1 -1
- package/dist/lib/store/hook/hookManager.d.ts.map +1 -1
- package/dist/lib/store/hook/hookManager.js +19 -8
- package/dist/lib/store/store.d.ts.map +1 -1
- package/dist/lib/store/store.js +9 -4
- package/dist/lib/store/store.types.d.ts +14 -37
- package/dist/lib/store/store.types.d.ts.map +1 -1
- package/dist/lib/store/storeManager.d.ts.map +1 -1
- package/dist/lib/store/storeManager.js +20 -10
- package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts +16 -0
- package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts.map +1 -0
- package/dist/lib/store/workDir/storage/{getChangedLinesSince.js → getChangedLinesBetween.js} +22 -20
- package/dist/lib/store/workDir/storage/getDataFromLogLine.d.ts.map +1 -1
- package/dist/lib/store/workDir/storage/getDataFromLogLine.js +16 -3
- package/dist/lib/store/workDir/storage/index.d.ts +1 -1
- package/dist/lib/store/workDir/storage/index.d.ts.map +1 -1
- package/dist/lib/store/workDir/storage/index.js +1 -1
- package/dist/lib/store/workDir/workDir.types.d.ts +9 -1
- package/dist/lib/store/workDir/workDir.types.d.ts.map +1 -1
- package/dist/lib/store/workDir/workDirHelper.d.ts +13 -6
- package/dist/lib/store/workDir/workDirHelper.d.ts.map +1 -1
- package/dist/lib/store/workDir/workDirHelper.js +42 -11
- package/dist/lib/utils/fs/fsUtils.d.ts +6 -0
- package/dist/lib/utils/fs/fsUtils.d.ts.map +1 -1
- package/dist/lib/utils/fs/fsUtils.js +20 -1
- package/dist/lib/utils/module/moduleManager.d.ts.map +1 -1
- package/dist/lib/utils/module/moduleManager.js +5 -1
- package/dist/lib/utils/string/stringUtils.d.ts +14 -0
- package/dist/lib/utils/string/stringUtils.d.ts.map +1 -1
- package/dist/lib/utils/string/stringUtils.js +28 -1
- package/dist/static-suite-data-server.d.ts +19 -39
- package/package.json +3 -3
- package/src/http/routes/dependency.ts +91 -0
- package/src/http/routes/dump.ts +21 -10
- package/src/http/routes/home.ts +2 -2
- package/src/http/routes/index.ts +16 -0
- package/src/http/routes/status.ts +1 -9
- package/src/http/views/dependencyIndex.pug +14 -0
- package/src/lib/dataServer.ts +2 -0
- package/src/lib/query/query.types.ts +2 -2
- package/src/lib/query/queryRunner.ts +3 -1
- package/src/lib/store/dataDir/dataDir.types.ts +5 -5
- package/src/lib/store/dataDir/dataDirManager.ts +30 -14
- package/src/lib/store/dependency/dependency.types.ts +0 -5
- package/src/lib/store/dependency/dependencyFileHelper.ts +5 -3
- package/src/lib/store/dependency/dependencyManager.ts +6 -3
- package/src/lib/store/dependency/dependencyTagger.ts +3 -39
- package/src/lib/store/diff/{diffManager.types.ts → diff.types.ts} +14 -5
- package/src/lib/store/diff/diffManager.ts +45 -84
- package/src/lib/store/dump/{dumpManager.types.ts → dump.types.ts} +24 -7
- package/src/lib/store/dump/dumpManager.ts +40 -75
- package/src/lib/store/dump/dumpMetadataHelper.ts +73 -0
- package/src/lib/store/hook/hook.types.ts +58 -23
- package/src/lib/store/hook/hookManager.ts +24 -9
- package/src/lib/store/store.ts +9 -4
- package/src/lib/store/store.types.ts +16 -37
- package/src/lib/store/storeManager.ts +28 -12
- package/src/lib/store/workDir/__tests__/workDirHelper.test.ts +6 -1
- package/src/lib/store/workDir/storage/__tests__/{getChangedLinesSince.test.ts → getChangedLinesBetween.test.ts} +9 -4
- package/src/lib/store/workDir/storage/getChangedLinesBetween.ts +52 -0
- package/src/lib/store/workDir/storage/getDataFromLogLine.ts +22 -3
- package/src/lib/store/workDir/storage/index.ts +1 -1
- package/src/lib/store/workDir/workDir.types.ts +11 -1
- package/src/lib/store/workDir/workDirHelper.ts +45 -11
- package/src/lib/utils/fs/fsUtils.ts +18 -0
- package/src/lib/utils/module/moduleManager.ts +5 -1
- package/src/lib/utils/string/__tests__/stringUtils.test.ts +30 -1
- package/src/lib/utils/string/stringUtils.ts +32 -0
- package/dist/lib/query/queryRunner copy.d.ts +0 -3
- package/dist/lib/query/queryRunner copy.d.ts.map +0 -1
- package/dist/lib/query/queryRunner copy.js +0 -87
- package/dist/lib/store/dependency/dependencyManager copy.d.ts +0 -6
- package/dist/lib/store/dependency/dependencyManager copy.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyManager copy.js +0 -91
- package/dist/lib/store/dependency/dependencyManager.types.d.ts +0 -48
- package/dist/lib/store/dependency/dependencyManager.types.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyTagManager.d.ts +0 -6
- package/dist/lib/store/dependency/dependencyTagManager.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyTagManager.js +0 -33
- package/dist/lib/store/diff/diffManager.types.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.d.ts +0 -3
- package/dist/lib/store/diff/tracker.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.js +0 -82
- package/dist/lib/store/diff/tracker.types.d.ts +0 -31
- package/dist/lib/store/diff/tracker.types.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.types.js +0 -2
- package/dist/lib/store/dump/dumpManager.types.d.ts.map +0 -1
- package/dist/lib/store/dump/dumpManager.types.js +0 -2
- package/dist/lib/store/include/includeIndex.d.ts +0 -6
- package/dist/lib/store/include/includeIndex.d.ts.map +0 -1
- package/dist/lib/store/include/includeIndex.js +0 -65
- package/dist/lib/store/include/includeIndex.types.d.ts +0 -29
- package/dist/lib/store/include/includeIndex.types.d.ts.map +0 -1
- package/dist/lib/store/include/includeIndex.types.js +0 -2
- package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts +0 -9
- package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts.map +0 -1
- package/src/lib/store/workDir/storage/getChangedLinesSince.ts +0 -47
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export declare type DiffManager = {
|
|
5
5
|
/**
|
|
6
|
-
* Clears intermediate changes tracked by
|
|
6
|
+
* Clears intermediate changes tracked by dependency manager.
|
|
7
7
|
*
|
|
8
|
-
* @param
|
|
8
|
+
* @param uniqueId - Unique id to be used for the next diff.
|
|
9
9
|
*/
|
|
10
|
-
reset(
|
|
10
|
+
reset(uniqueId: string): void;
|
|
11
11
|
/**
|
|
12
12
|
* Gets a list of changed files affected by includes and queries.
|
|
13
13
|
*
|
|
@@ -35,9 +35,17 @@ export declare type DiffManager = {
|
|
|
35
35
|
*/
|
|
36
36
|
export declare type Diff = {
|
|
37
37
|
/**
|
|
38
|
-
*
|
|
38
|
+
* Execution time taken by the diff, in milliseconds.
|
|
39
39
|
*/
|
|
40
|
-
|
|
40
|
+
execTimeMs: number;
|
|
41
|
+
/**
|
|
42
|
+
* A unique id representing the date from which this diff is obtained.
|
|
43
|
+
*/
|
|
44
|
+
fromUniqueId: string;
|
|
45
|
+
/**
|
|
46
|
+
* A unique id representing the date until which this diff is obtained.
|
|
47
|
+
*/
|
|
48
|
+
toUniqueId: string;
|
|
41
49
|
/**
|
|
42
50
|
* A list of updated files (both newly added and changed files).
|
|
43
51
|
*/
|
|
@@ -47,4 +55,4 @@ export declare type Diff = {
|
|
|
47
55
|
*/
|
|
48
56
|
deleted: Set<string>;
|
|
49
57
|
};
|
|
50
|
-
//# sourceMappingURL=
|
|
58
|
+
//# sourceMappingURL=diff.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diff.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,oBAAY,WAAW,GAAG;IACxB;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B;;;;;;;;;;;;;;;;;OAiBG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;CACnD,CAAC;AAEF;;GAEG;AACH,oBAAY,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAErB;;OAEG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACtB,CAAC"}
|
|
File without changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diffManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diffManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAQ,MAAM,
|
|
1
|
+
{"version":3,"file":"diffManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/diff/diffManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAQ,MAAM,cAAc,CAAC;AAQjD,eAAO,MAAM,WAAW,EAAE,WAuEzB,CAAC"}
|
|
@@ -9,106 +9,63 @@ const logger_1 = require("@lib/utils/logger");
|
|
|
9
9
|
const store_1 = require("../store");
|
|
10
10
|
const dataDirManager_1 = require("../dataDir/dataDirManager");
|
|
11
11
|
const dependencyManager_1 = require("../dependency/dependencyManager");
|
|
12
|
-
|
|
12
|
+
const workDir_1 = require("../workDir");
|
|
13
|
+
const dumpMetadataHelper_1 = require("../dump/dumpMetadataHelper");
|
|
14
|
+
let lastDiffUniqueId = workDir_1.unixEpochUniqueId;
|
|
13
15
|
exports.diffManager = {
|
|
14
|
-
reset(
|
|
15
|
-
|
|
16
|
+
reset(uniqueId) {
|
|
17
|
+
lastDiffUniqueId = uniqueId;
|
|
16
18
|
dependencyManager_1.dependencyManager.reset();
|
|
17
19
|
},
|
|
18
20
|
getDiff(options = { incremental: true }) {
|
|
19
21
|
const startDate = microtime_1.default.now();
|
|
20
22
|
// Before getting any diff data, update any pending changes from data dir.
|
|
21
|
-
dataDirManager_1.dataDirManager.update();
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
const changedFiles = dataDirManager_1.dataDirManager.update();
|
|
24
|
+
const updated = new Set();
|
|
25
|
+
const deleted = new Set();
|
|
26
|
+
const currentDumpUniqueId = dumpMetadataHelper_1.dumpMetadataHelper.getCurrentDumpUniqueId();
|
|
27
|
+
const isFirstDiffAfterReboot = lastDiffUniqueId === workDir_1.unixEpochUniqueId;
|
|
28
|
+
const isADumpAvailable = currentDumpUniqueId !== workDir_1.unixEpochUniqueId;
|
|
29
|
+
// If both checks passes, check if something has changed after last dump to avoid a full diff:
|
|
30
|
+
if (isFirstDiffAfterReboot && isADumpAvailable) {
|
|
31
|
+
const dataDirModificationUniqueId = changedFiles.toUniqueId;
|
|
32
|
+
if (currentDumpUniqueId === store_1.store.initialUniqueId) {
|
|
33
|
+
// If nothing changed, use store.initialUniqueId as lastDiffUniqueId, to
|
|
34
|
+
// execute an incremental dump.
|
|
35
|
+
lastDiffUniqueId = store_1.store.initialUniqueId;
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
// If something changed, keep lastDiffUniqueId as is, and get the list of
|
|
39
|
+
// changed files from last dump. From that list, we need deleted files, since
|
|
40
|
+
// those files are no more in data dir and we need to remove them from dump dir.
|
|
41
|
+
const changedFilesSinceLastDump = workDir_1.workDirHelper.getChangedFilesBetween(currentDumpUniqueId, dataDirModificationUniqueId);
|
|
42
|
+
changedFilesSinceLastDump.deleted.forEach(filepath => deleted.add(filepath));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (options.incremental && lastDiffUniqueId !== workDir_1.unixEpochUniqueId) {
|
|
46
|
+
logger_1.logger.debug(`Getting incremental diff using date "${lastDiffUniqueId}"`);
|
|
26
47
|
const invalidatedFilepaths = dependencyManager_1.dependencyManager.getInvalidatedFilepaths();
|
|
27
|
-
updated
|
|
28
|
-
deleted
|
|
48
|
+
invalidatedFilepaths.updated.forEach(filepath => updated.add(filepath));
|
|
49
|
+
invalidatedFilepaths.deleted.forEach(filepath => deleted.add(filepath));
|
|
29
50
|
}
|
|
30
51
|
else {
|
|
31
|
-
//
|
|
52
|
+
// Return all files
|
|
32
53
|
logger_1.logger.debug(`Getting full diff with no date`);
|
|
33
|
-
|
|
54
|
+
Array.from(store_1.store.data.keys()).forEach(key => updated.add(key));
|
|
34
55
|
}
|
|
56
|
+
const execTimeMs = (microtime_1.default.now() - startDate) / 1000;
|
|
35
57
|
const diff = {
|
|
36
|
-
|
|
58
|
+
execTimeMs,
|
|
59
|
+
fromUniqueId: lastDiffUniqueId,
|
|
60
|
+
toUniqueId: changedFiles.toUniqueId,
|
|
37
61
|
updated,
|
|
38
62
|
deleted,
|
|
39
63
|
};
|
|
40
|
-
logger_1.logger.info(`Diff created in ${
|
|
64
|
+
logger_1.logger.info(`Diff created in ${execTimeMs} ms. Updated: ${diff.updated.size} / Deleted: ${diff.deleted.size}`);
|
|
41
65
|
// Log diff if not empty
|
|
42
66
|
// if (diff.updated.length || diff.deleted.length) {
|
|
43
67
|
// logger.debug(`Diff: "${JSON.stringify(jsonify(diff))}"`);
|
|
44
68
|
// }
|
|
45
69
|
return diff;
|
|
46
70
|
},
|
|
47
|
-
/*
|
|
48
|
-
getDiff(options = { incremental: true }): Diff {
|
|
49
|
-
const startDate = microtime.now();
|
|
50
|
-
|
|
51
|
-
// Before getting any diff data, update any pending changes from data dir.
|
|
52
|
-
const changedFiles = dataDirManager.update();
|
|
53
|
-
|
|
54
|
-
let updated = new Set<string>();
|
|
55
|
-
let deleted = new Set<string>();
|
|
56
|
-
const sinceDate = getLastDiffDate();
|
|
57
|
-
if (options.incremental && sinceDate) {
|
|
58
|
-
logger.debug(`Getting incremental diff using date "${sinceDate}"`);
|
|
59
|
-
// Only process changedFiles if not empty.
|
|
60
|
-
if (changedFiles.updated.length || changedFiles.deleted.length) {
|
|
61
|
-
// Update tracked files so no affected parent is missed.
|
|
62
|
-
changedFiles.updated.forEach(file => {
|
|
63
|
-
tracker.trackChangedFile(file);
|
|
64
|
-
});
|
|
65
|
-
changedFiles.deleted.forEach(file => {
|
|
66
|
-
tracker.trackChangedFile(file);
|
|
67
|
-
});
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// Create the resulting set of updated and deleted files.
|
|
71
|
-
// "updated" includes all affected parents tracked down
|
|
72
|
-
// by tracker, without any deleted file.
|
|
73
|
-
updated = new Set<string>(tracker.getChangedFiles());
|
|
74
|
-
changedFiles.deleted.forEach(file => updated.delete(file));
|
|
75
|
-
|
|
76
|
-
// "deleted" includes only deleted files, and is based only
|
|
77
|
-
// on data coming from changedFiles, because all deleted files
|
|
78
|
-
// are stored on Static Suite Data Server log
|
|
79
|
-
deleted = new Set<string>(changedFiles.deleted);
|
|
80
|
-
|
|
81
|
-
// Add updated files affected by queries only if "updated" or "deleted"
|
|
82
|
-
// contain any changes.
|
|
83
|
-
if (updated.size > 0 || deleted.size > 0) {
|
|
84
|
-
queryTagManager
|
|
85
|
-
.getInvalidFilepaths()
|
|
86
|
-
.forEach(file => updated.add(file));
|
|
87
|
-
queryTagManager.resetInvalidatedTags();
|
|
88
|
-
}
|
|
89
|
-
} else {
|
|
90
|
-
// If no sinceDate is passed, return all files
|
|
91
|
-
logger.debug(`Getting full diff with no date`);
|
|
92
|
-
updated = new Set<string>(store.data.keys());
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
const diff: Diff = {
|
|
96
|
-
since: sinceDate ? sinceDate.getTime() : new Date(0).getTime(),
|
|
97
|
-
updated,
|
|
98
|
-
deleted,
|
|
99
|
-
};
|
|
100
|
-
|
|
101
|
-
logger.info(
|
|
102
|
-
`Diff created in ${(microtime.now() - startDate) / 1000} ms. Updated: ${
|
|
103
|
-
diff.updated.size
|
|
104
|
-
} / Deleted: ${diff.deleted.size}`,
|
|
105
|
-
);
|
|
106
|
-
|
|
107
|
-
// Log diff if not empty
|
|
108
|
-
// if (diff.updated.length || diff.deleted.length) {
|
|
109
|
-
// logger.debug(`Diff: "${JSON.stringify(jsonify(diff))}"`);
|
|
110
|
-
// }
|
|
111
|
-
|
|
112
|
-
return diff;
|
|
113
|
-
}, */
|
|
114
71
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Diff } from '../diff/diff.types';
|
|
1
2
|
/**
|
|
2
3
|
* Service that dumps resolved data (with resolved includes) into a dump directory.
|
|
3
4
|
*/
|
|
@@ -21,20 +22,28 @@ export declare type DumpManager = {
|
|
|
21
22
|
* Reset dump metadata.
|
|
22
23
|
*
|
|
23
24
|
* @remarks
|
|
24
|
-
* Resets dump metadata, removing any dump information older than the passed
|
|
25
|
+
* Resets dump metadata, removing any dump information older than the passed unique id.
|
|
25
26
|
*
|
|
26
|
-
* @param
|
|
27
|
+
* @param uniqueId - Unique id
|
|
27
28
|
*/
|
|
28
|
-
reset(
|
|
29
|
+
reset(uniqueId: string): void;
|
|
29
30
|
};
|
|
30
31
|
/**
|
|
31
32
|
* A group of files to by dumped to dump directory.
|
|
32
33
|
*/
|
|
33
34
|
export declare type Dump = {
|
|
34
35
|
/**
|
|
35
|
-
*
|
|
36
|
+
* Execution time taken by the dump, in milliseconds.
|
|
36
37
|
*/
|
|
37
|
-
|
|
38
|
+
execTimeMs: number;
|
|
39
|
+
/**
|
|
40
|
+
* A unique id representing the date from which this dump is obtained.
|
|
41
|
+
*/
|
|
42
|
+
fromUniqueId: string;
|
|
43
|
+
/**
|
|
44
|
+
* A unique id representing the date until which this dump is obtained.
|
|
45
|
+
*/
|
|
46
|
+
toUniqueId: string;
|
|
38
47
|
/**
|
|
39
48
|
* A list of updated files, keyed by file path. Its value is an object with two keys,
|
|
40
49
|
* old public URL and new public URL (the value stored at data.content.url.path) before
|
|
@@ -55,8 +64,12 @@ export declare type Dump = {
|
|
|
55
64
|
newPublicUrl: null;
|
|
56
65
|
}>;
|
|
57
66
|
/**
|
|
58
|
-
*
|
|
67
|
+
* The diff used to generate this dump.
|
|
59
68
|
*/
|
|
60
|
-
|
|
69
|
+
diff: Diff;
|
|
70
|
+
};
|
|
71
|
+
export declare type DumpMetadata = {
|
|
72
|
+
current: string;
|
|
73
|
+
dumps: Dump[];
|
|
61
74
|
};
|
|
62
|
-
//# sourceMappingURL=
|
|
75
|
+
//# sourceMappingURL=dump.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dump.types.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dump.types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAE1C;;GAEG;AACH,oBAAY,WAAW,GAAG;IACxB;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAE/C;;;;;;;OAOG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEF;;GAEG;AACH,oBAAY,IAAI,GAAG;IACjB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,OAAO,EAAE,GAAG,CACV,MAAM,EACN;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAC7D,CAAC;IAEF;;;;;OAKG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,IAAI,CAAA;KAAE,CAAC,CAAC;IAE1E;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,oBAAY,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf,CAAC"}
|
|
File without changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dumpManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpManager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dumpManager.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpManager.ts"],"names":[],"mappings":"AAcA,OAAO,EAAQ,WAAW,EAAE,MAAM,cAAc,CAAC;AA2GjD,eAAO,MAAM,WAAW,EAAE,WA8EzB,CAAC"}
|
|
@@ -8,25 +8,12 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
9
|
const microtime_1 = __importDefault(require("microtime"));
|
|
10
10
|
const logger_1 = require("@lib/utils/logger");
|
|
11
|
-
const object_1 = require("@lib/utils/object");
|
|
12
11
|
const fs_2 = require("@lib/utils/fs");
|
|
13
12
|
const config_1 = require("@lib/config");
|
|
14
13
|
const store_1 = require("@lib/store");
|
|
15
14
|
const diffManager_1 = require("../diff/diffManager");
|
|
16
15
|
const hook_1 = require("../hook");
|
|
17
|
-
const
|
|
18
|
-
const isEmpty = fs_1.default.readdirSync(dir).length === 0;
|
|
19
|
-
if (isEmpty) {
|
|
20
|
-
try {
|
|
21
|
-
fs_1.default.rmdirSync(dir);
|
|
22
|
-
}
|
|
23
|
-
catch (e) {
|
|
24
|
-
logger_1.logger.debug(`Error deleting empty directory "${dir}": ${e}`);
|
|
25
|
-
}
|
|
26
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
27
|
-
removeEmptyDirsUpwards(path_1.default.dirname(dir));
|
|
28
|
-
}
|
|
29
|
-
};
|
|
16
|
+
const dumpMetadataHelper_1 = require("./dumpMetadataHelper");
|
|
30
17
|
const storeUpdatedFiles = (diff, dump, dumpDir) => {
|
|
31
18
|
diff.updated.forEach(relativeFilepath => {
|
|
32
19
|
const storeFileContentData = store_1.store.data.get(relativeFilepath);
|
|
@@ -60,9 +47,19 @@ const storeUpdatedFiles = (diff, dump, dumpDir) => {
|
|
|
60
47
|
JSON.parse(dumpFileContentString)?.data?.content?.url?.path ||
|
|
61
48
|
null;
|
|
62
49
|
/*
|
|
50
|
+
// Save a copy for later reference.
|
|
51
|
+
const backupAbsoluteFilepathInDumpDir =
|
|
52
|
+
absoluteFilepathInDumpDir.replace(
|
|
53
|
+
'/dump/files/',
|
|
54
|
+
'/dump/files.bak/',
|
|
55
|
+
);
|
|
56
|
+
const backupDir = path.dirname(backupAbsoluteFilepathInDumpDir);
|
|
57
|
+
if (!fs.existsSync(backupDir)) {
|
|
58
|
+
fs.mkdirSync(backupDir, { recursive: true });
|
|
59
|
+
}
|
|
63
60
|
fs.renameSync(
|
|
64
61
|
absoluteFilepathInDumpDir,
|
|
65
|
-
|
|
62
|
+
backupAbsoluteFilepathInDumpDir,
|
|
66
63
|
);
|
|
67
64
|
*/
|
|
68
65
|
}
|
|
@@ -96,6 +93,8 @@ const removeDeletedFiles = (diff, dump, dumpDir) => {
|
|
|
96
93
|
const oldPublicUrl = dumpContent.json?.data?.content?.url?.path;
|
|
97
94
|
// Delete it.
|
|
98
95
|
fs_1.default.unlinkSync(absoluteFilepath);
|
|
96
|
+
// Delete any empty directory.
|
|
97
|
+
(0, fs_2.removeEmptyDirsUpwards)(path_1.default.dirname(absoluteFilepath));
|
|
99
98
|
// Mark it as deleted.
|
|
100
99
|
dump.deleted.set(relativeFilepath, {
|
|
101
100
|
oldPublicUrl,
|
|
@@ -108,41 +107,21 @@ const removeDeletedFiles = (diff, dump, dumpDir) => {
|
|
|
108
107
|
}
|
|
109
108
|
});
|
|
110
109
|
};
|
|
111
|
-
const storeDumpMetadata = (metadataFilepath, dump, diffResetDate) => {
|
|
112
|
-
let currentDumpMetadata = [];
|
|
113
|
-
try {
|
|
114
|
-
currentDumpMetadata = JSON.parse(fs_1.default.readFileSync(metadataFilepath).toString());
|
|
115
|
-
}
|
|
116
|
-
catch (e) {
|
|
117
|
-
currentDumpMetadata = [];
|
|
118
|
-
}
|
|
119
|
-
currentDumpMetadata.push((0, object_1.jsonify)(dump));
|
|
120
|
-
try {
|
|
121
|
-
const currentDumpMetadataString = JSON.stringify(currentDumpMetadata);
|
|
122
|
-
fs_1.default.writeFileSync(metadataFilepath, currentDumpMetadataString);
|
|
123
|
-
// Resetting the diff must happen when dump metadata is successfully
|
|
124
|
-
// stored into disk.
|
|
125
|
-
diffManager_1.diffManager.reset(diffResetDate);
|
|
126
|
-
}
|
|
127
|
-
catch (e) {
|
|
128
|
-
logger_1.logger.error(`Dump: error saving dump metadata to "${metadataFilepath}": ${e}`);
|
|
129
|
-
}
|
|
130
|
-
};
|
|
131
110
|
exports.dumpManager = {
|
|
132
111
|
dump(options = { incremental: true }) {
|
|
133
112
|
const startDate = microtime_1.default.now();
|
|
134
|
-
// Create a diff reset date just before consuming a diff.
|
|
135
|
-
const diffResetDate = new Date();
|
|
136
|
-
const diff = diffManager_1.diffManager.getDiff({ incremental: options.incremental });
|
|
137
113
|
// Diff data is processed and transformed into a dump object.
|
|
114
|
+
const diff = diffManager_1.diffManager.getDiff({ incremental: options.incremental });
|
|
138
115
|
let dump = {
|
|
139
|
-
|
|
116
|
+
execTimeMs: 0,
|
|
117
|
+
fromUniqueId: diff.fromUniqueId,
|
|
118
|
+
toUniqueId: diff.toUniqueId,
|
|
140
119
|
updated: new Map(),
|
|
141
120
|
deleted: new Map(),
|
|
121
|
+
diff,
|
|
142
122
|
};
|
|
143
123
|
if (config_1.config.dumpDir) {
|
|
144
124
|
const dumpDir = `${config_1.config.dumpDir}/files`;
|
|
145
|
-
const metadataFilepath = `${config_1.config.dumpDir}/metadata.json`;
|
|
146
125
|
if (diff.updated.size || diff.deleted.size) {
|
|
147
126
|
// Store updated files.
|
|
148
127
|
storeUpdatedFiles(diff, dump, dumpDir);
|
|
@@ -154,16 +133,23 @@ exports.dumpManager = {
|
|
|
154
133
|
dump.execTimeMs = execTimeMs;
|
|
155
134
|
// Merge and store dump metadata if any.
|
|
156
135
|
if (dump.updated.size || dump.deleted.size) {
|
|
157
|
-
storeDumpMetadata(
|
|
136
|
+
const storeSuccessful = dumpMetadataHelper_1.dumpMetadataHelper.storeDumpMetadata(dump);
|
|
137
|
+
if (storeSuccessful) {
|
|
138
|
+
// Resetting the diff must happen when dump metadata is successfully
|
|
139
|
+
// stored into disk.
|
|
140
|
+
diffManager_1.diffManager.reset(diff.toUniqueId);
|
|
141
|
+
}
|
|
158
142
|
logger_1.logger.info(`Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`);
|
|
143
|
+
/*
|
|
159
144
|
// Log dump if not empty
|
|
160
145
|
if (dump.updated.size || dump.deleted.size) {
|
|
161
|
-
|
|
146
|
+
logger.debug(`Dump: "${JSON.stringify(jsonify(dump))}"`);
|
|
162
147
|
}
|
|
148
|
+
*/
|
|
163
149
|
}
|
|
164
150
|
else {
|
|
165
151
|
// Resetting the diff must happen when no other operations are pending.
|
|
166
|
-
diffManager_1.diffManager.reset(
|
|
152
|
+
diffManager_1.diffManager.reset(diff.toUniqueId);
|
|
167
153
|
logger_1.logger.info(`Dump done in ${execTimeMs} ms without changes stored into disk.`);
|
|
168
154
|
}
|
|
169
155
|
}
|
|
@@ -176,21 +162,10 @@ exports.dumpManager = {
|
|
|
176
162
|
}
|
|
177
163
|
return dump;
|
|
178
164
|
},
|
|
179
|
-
reset(
|
|
165
|
+
reset(uniqueId) {
|
|
180
166
|
if (config_1.config.dumpDir) {
|
|
181
|
-
const
|
|
182
|
-
const metadata = JSON.parse(fs_1.default.existsSync(metadataFilepath)
|
|
183
|
-
? fs_1.default.readFileSync(metadataFilepath).toString()
|
|
184
|
-
: '[]');
|
|
185
|
-
// Remove any dump data older that timestamp
|
|
186
|
-
const resetMetadata = metadata.filter((dump) => dump.since > timestamp);
|
|
167
|
+
const resetMetadata = dumpMetadataHelper_1.dumpMetadataHelper.removeDumpDataOlderThan(uniqueId);
|
|
187
168
|
logger_1.logger.debug(`Dump reset : "${JSON.stringify(resetMetadata)}"`);
|
|
188
|
-
try {
|
|
189
|
-
fs_1.default.writeFileSync(metadataFilepath, JSON.stringify(resetMetadata));
|
|
190
|
-
}
|
|
191
|
-
catch (e) {
|
|
192
|
-
logger_1.logger.error(`Dump: error resetting dump metadata to "${metadataFilepath}": ${e}`);
|
|
193
|
-
}
|
|
194
169
|
}
|
|
195
170
|
else {
|
|
196
171
|
logger_1.logger.error('"dumpDir" option not provided. Dump reset cannot be executed.');
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Dump, DumpMetadata } from './dump.types';
|
|
2
|
+
export declare const dumpMetadataHelper: {
|
|
3
|
+
storeDumpMetadata: (dump: Dump) => boolean;
|
|
4
|
+
removeDumpDataOlderThan(uniqueId: string): DumpMetadata;
|
|
5
|
+
getCurrentDumpUniqueId(): string;
|
|
6
|
+
};
|
|
7
|
+
//# sourceMappingURL=dumpMetadataHelper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dumpMetadataHelper.d.ts","sourceRoot":"","sources":["../../../../src/lib/store/dump/dumpMetadataHelper.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAKlD,eAAO,MAAM,kBAAkB;8BACH,IAAI,KAAG,OAAO;sCA4BN,MAAM,GAAG,YAAY;8BAuB7B,MAAM;CAWjC,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.dumpMetadataHelper = void 0;
|
|
7
|
+
const fs_1 = __importDefault(require("fs"));
|
|
8
|
+
const logger_1 = require("@lib/utils/logger");
|
|
9
|
+
const object_1 = require("@lib/utils/object");
|
|
10
|
+
const config_1 = require("@lib/config");
|
|
11
|
+
const workDir_1 = require("../workDir");
|
|
12
|
+
const getMetadataFilepath = () => `${config_1.config.dumpDir}/metadata.json`;
|
|
13
|
+
exports.dumpMetadataHelper = {
|
|
14
|
+
storeDumpMetadata: (dump) => {
|
|
15
|
+
const metadataFilepath = getMetadataFilepath();
|
|
16
|
+
const fallbackDumpMetadata = {
|
|
17
|
+
current: workDir_1.unixEpochUniqueId,
|
|
18
|
+
dumps: [],
|
|
19
|
+
};
|
|
20
|
+
let currentDumpMetadata = fallbackDumpMetadata;
|
|
21
|
+
try {
|
|
22
|
+
currentDumpMetadata = JSON.parse(fs_1.default.readFileSync(metadataFilepath).toString());
|
|
23
|
+
}
|
|
24
|
+
catch (e) {
|
|
25
|
+
currentDumpMetadata = fallbackDumpMetadata;
|
|
26
|
+
}
|
|
27
|
+
currentDumpMetadata.current = dump.toUniqueId;
|
|
28
|
+
currentDumpMetadata.dumps.push((0, object_1.jsonify)(dump));
|
|
29
|
+
try {
|
|
30
|
+
const currentDumpMetadataString = JSON.stringify(currentDumpMetadata);
|
|
31
|
+
fs_1.default.writeFileSync(metadataFilepath, currentDumpMetadataString);
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
catch (e) {
|
|
35
|
+
logger_1.logger.error(`Dump: error saving dump metadata to "${metadataFilepath}": ${e}`);
|
|
36
|
+
}
|
|
37
|
+
return false;
|
|
38
|
+
},
|
|
39
|
+
removeDumpDataOlderThan(uniqueId) {
|
|
40
|
+
const metadataFilepath = getMetadataFilepath();
|
|
41
|
+
const metadata = JSON.parse(fs_1.default.existsSync(metadataFilepath)
|
|
42
|
+
? fs_1.default.readFileSync(metadataFilepath).toString()
|
|
43
|
+
: '[]');
|
|
44
|
+
// Remove any dump data older than or equal to unique id.
|
|
45
|
+
metadata.dumps = metadata.dumps.filter(dump => dump.toUniqueId > uniqueId);
|
|
46
|
+
try {
|
|
47
|
+
fs_1.default.writeFileSync(metadataFilepath, JSON.stringify(metadata));
|
|
48
|
+
}
|
|
49
|
+
catch (e) {
|
|
50
|
+
logger_1.logger.error(`Dump: error removing dump metadata older than "${uniqueId}" from "${metadataFilepath}": ${e}`);
|
|
51
|
+
}
|
|
52
|
+
return metadata;
|
|
53
|
+
},
|
|
54
|
+
getCurrentDumpUniqueId() {
|
|
55
|
+
const metadataFilepath = getMetadataFilepath();
|
|
56
|
+
const metadata = JSON.parse(fs_1.default.existsSync(metadataFilepath)
|
|
57
|
+
? fs_1.default.readFileSync(metadataFilepath).toString()
|
|
58
|
+
: '[]');
|
|
59
|
+
return metadata.current || workDir_1.unixEpochUniqueId;
|
|
60
|
+
},
|
|
61
|
+
};
|