@static-suite/static-suite-data-server 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +569 -107
- package/dist/http/README.md +569 -107
- package/dist/http/routes/dependency.d.ts +10 -0
- package/dist/http/routes/dependency.d.ts.map +1 -0
- package/dist/http/routes/dependency.js +71 -0
- package/dist/http/routes/dump.d.ts.map +1 -1
- package/dist/http/routes/dump.js +15 -10
- package/dist/http/routes/home.js +2 -1
- package/dist/http/routes/index.d.ts.map +1 -1
- package/dist/http/routes/index.js +8 -0
- package/dist/http/routes/status.d.ts.map +1 -1
- package/dist/http/routes/status.js +1 -6
- package/dist/http/views/dependencyIndex.pug +14 -0
- package/dist/lib/dataServer.d.ts +2 -1
- package/dist/lib/dataServer.d.ts.map +1 -1
- package/dist/lib/query/query.types.d.ts +2 -2
- package/dist/lib/query/query.types.d.ts.map +1 -1
- package/dist/lib/query/query.types.js +2 -2
- package/dist/lib/query/queryRunner.d.ts.map +1 -1
- package/dist/lib/query/queryRunner.js +3 -1
- package/dist/lib/store/dataDir/dataDir.types.d.ts +5 -5
- package/dist/lib/store/dataDir/dataDir.types.d.ts.map +1 -1
- package/dist/lib/store/dataDir/dataDirManager.d.ts.map +1 -1
- package/dist/lib/store/dataDir/dataDirManager.js +23 -12
- package/dist/lib/store/dependency/dependency.types.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.d.ts +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyFileHelper.js +5 -3
- package/dist/lib/store/dependency/dependencyManager.d.ts +12 -0
- package/dist/lib/store/dependency/dependencyManager.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyManager.js +7 -5
- package/dist/lib/store/dependency/dependencyTagger.d.ts.map +1 -1
- package/dist/lib/store/dependency/dependencyTagger.js +3 -38
- package/dist/lib/store/diff/{diffManager.types.d.ts → diff.types.d.ts} +14 -6
- package/dist/lib/store/diff/diff.types.d.ts.map +1 -0
- package/dist/lib/store/{dependency/dependencyManager.types.js → diff/diff.types.js} +0 -0
- package/dist/lib/store/diff/diffManager.d.ts +1 -1
- package/dist/lib/store/diff/diffManager.d.ts.map +1 -1
- package/dist/lib/store/diff/diffManager.js +38 -81
- package/dist/lib/store/dump/{dumpManager.types.d.ts → dump.types.d.ts} +21 -8
- package/dist/lib/store/dump/dump.types.d.ts.map +1 -0
- package/dist/lib/store/{diff/diffManager.types.js → dump/dump.types.js} +0 -0
- package/dist/lib/store/dump/dumpManager.d.ts +1 -1
- package/dist/lib/store/dump/dumpManager.d.ts.map +1 -1
- package/dist/lib/store/dump/dumpManager.js +31 -56
- package/dist/lib/store/dump/dumpMetadataHelper.d.ts +7 -0
- package/dist/lib/store/dump/dumpMetadataHelper.d.ts.map +1 -0
- package/dist/lib/store/dump/dumpMetadataHelper.js +61 -0
- package/dist/lib/store/hook/hook.types.d.ts +49 -23
- package/dist/lib/store/hook/hook.types.d.ts.map +1 -1
- package/dist/lib/store/hook/hookManager.d.ts.map +1 -1
- package/dist/lib/store/hook/hookManager.js +19 -8
- package/dist/lib/store/store.d.ts.map +1 -1
- package/dist/lib/store/store.js +9 -4
- package/dist/lib/store/store.types.d.ts +14 -37
- package/dist/lib/store/store.types.d.ts.map +1 -1
- package/dist/lib/store/storeManager.d.ts.map +1 -1
- package/dist/lib/store/storeManager.js +20 -10
- package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts +16 -0
- package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts.map +1 -0
- package/dist/lib/store/workDir/storage/{getChangedLinesSince.js → getChangedLinesBetween.js} +22 -20
- package/dist/lib/store/workDir/storage/getDataFromLogLine.d.ts.map +1 -1
- package/dist/lib/store/workDir/storage/getDataFromLogLine.js +16 -3
- package/dist/lib/store/workDir/storage/index.d.ts +1 -1
- package/dist/lib/store/workDir/storage/index.d.ts.map +1 -1
- package/dist/lib/store/workDir/storage/index.js +1 -1
- package/dist/lib/store/workDir/workDir.types.d.ts +9 -1
- package/dist/lib/store/workDir/workDir.types.d.ts.map +1 -1
- package/dist/lib/store/workDir/workDirHelper.d.ts +13 -6
- package/dist/lib/store/workDir/workDirHelper.d.ts.map +1 -1
- package/dist/lib/store/workDir/workDirHelper.js +42 -11
- package/dist/lib/utils/fs/fsUtils.d.ts +6 -0
- package/dist/lib/utils/fs/fsUtils.d.ts.map +1 -1
- package/dist/lib/utils/fs/fsUtils.js +20 -1
- package/dist/lib/utils/module/moduleManager.d.ts.map +1 -1
- package/dist/lib/utils/module/moduleManager.js +5 -1
- package/dist/lib/utils/string/stringUtils.d.ts +14 -0
- package/dist/lib/utils/string/stringUtils.d.ts.map +1 -1
- package/dist/lib/utils/string/stringUtils.js +28 -1
- package/dist/static-suite-data-server.d.ts +19 -39
- package/package.json +3 -3
- package/src/http/routes/dependency.ts +91 -0
- package/src/http/routes/dump.ts +21 -10
- package/src/http/routes/home.ts +2 -2
- package/src/http/routes/index.ts +16 -0
- package/src/http/routes/status.ts +1 -9
- package/src/http/views/dependencyIndex.pug +14 -0
- package/src/lib/dataServer.ts +2 -0
- package/src/lib/query/query.types.ts +2 -2
- package/src/lib/query/queryRunner.ts +3 -1
- package/src/lib/store/dataDir/dataDir.types.ts +5 -5
- package/src/lib/store/dataDir/dataDirManager.ts +30 -14
- package/src/lib/store/dependency/dependency.types.ts +0 -5
- package/src/lib/store/dependency/dependencyFileHelper.ts +5 -3
- package/src/lib/store/dependency/dependencyManager.ts +6 -3
- package/src/lib/store/dependency/dependencyTagger.ts +3 -39
- package/src/lib/store/diff/{diffManager.types.ts → diff.types.ts} +14 -5
- package/src/lib/store/diff/diffManager.ts +45 -84
- package/src/lib/store/dump/{dumpManager.types.ts → dump.types.ts} +24 -7
- package/src/lib/store/dump/dumpManager.ts +40 -75
- package/src/lib/store/dump/dumpMetadataHelper.ts +73 -0
- package/src/lib/store/hook/hook.types.ts +58 -23
- package/src/lib/store/hook/hookManager.ts +24 -9
- package/src/lib/store/store.ts +9 -4
- package/src/lib/store/store.types.ts +16 -37
- package/src/lib/store/storeManager.ts +28 -12
- package/src/lib/store/workDir/__tests__/workDirHelper.test.ts +6 -1
- package/src/lib/store/workDir/storage/__tests__/{getChangedLinesSince.test.ts → getChangedLinesBetween.test.ts} +9 -4
- package/src/lib/store/workDir/storage/getChangedLinesBetween.ts +52 -0
- package/src/lib/store/workDir/storage/getDataFromLogLine.ts +22 -3
- package/src/lib/store/workDir/storage/index.ts +1 -1
- package/src/lib/store/workDir/workDir.types.ts +11 -1
- package/src/lib/store/workDir/workDirHelper.ts +45 -11
- package/src/lib/utils/fs/fsUtils.ts +18 -0
- package/src/lib/utils/module/moduleManager.ts +5 -1
- package/src/lib/utils/string/__tests__/stringUtils.test.ts +30 -1
- package/src/lib/utils/string/stringUtils.ts +32 -0
- package/dist/lib/query/queryRunner copy.d.ts +0 -3
- package/dist/lib/query/queryRunner copy.d.ts.map +0 -1
- package/dist/lib/query/queryRunner copy.js +0 -87
- package/dist/lib/store/dependency/dependencyManager copy.d.ts +0 -6
- package/dist/lib/store/dependency/dependencyManager copy.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyManager copy.js +0 -91
- package/dist/lib/store/dependency/dependencyManager.types.d.ts +0 -48
- package/dist/lib/store/dependency/dependencyManager.types.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyTagManager.d.ts +0 -6
- package/dist/lib/store/dependency/dependencyTagManager.d.ts.map +0 -1
- package/dist/lib/store/dependency/dependencyTagManager.js +0 -33
- package/dist/lib/store/diff/diffManager.types.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.d.ts +0 -3
- package/dist/lib/store/diff/tracker.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.js +0 -82
- package/dist/lib/store/diff/tracker.types.d.ts +0 -31
- package/dist/lib/store/diff/tracker.types.d.ts.map +0 -1
- package/dist/lib/store/diff/tracker.types.js +0 -2
- package/dist/lib/store/dump/dumpManager.types.d.ts.map +0 -1
- package/dist/lib/store/dump/dumpManager.types.js +0 -2
- package/dist/lib/store/include/includeIndex.d.ts +0 -6
- package/dist/lib/store/include/includeIndex.d.ts.map +0 -1
- package/dist/lib/store/include/includeIndex.js +0 -65
- package/dist/lib/store/include/includeIndex.types.d.ts +0 -29
- package/dist/lib/store/include/includeIndex.types.d.ts.map +0 -1
- package/dist/lib/store/include/includeIndex.types.js +0 -2
- package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts +0 -9
- package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts.map +0 -1
- package/src/lib/store/workDir/storage/getChangedLinesSince.ts +0 -47
|
@@ -40,44 +40,18 @@ export interface StoreData<K, V> extends Map<K, V> {
|
|
|
40
40
|
*/
|
|
41
41
|
export type Store = {
|
|
42
42
|
/**
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
*
|
|
49
|
-
* - Data directory modified at 12:30:00
|
|
50
|
-
* - Syncing the store takes 3 seconds and finishes at 12:30:03
|
|
51
|
-
*
|
|
52
|
-
* ... syncDate will be 12:30:00, not 12:30:03.
|
|
53
|
-
*
|
|
54
|
-
* Doing it the other way round can cause data inconsistencies, because syncing files
|
|
55
|
-
* to the store takes time, it is done sequentially, and in the meanwhile, other
|
|
56
|
-
* modifications can happen:
|
|
57
|
-
* - Store last syncDate is 12:29:00
|
|
58
|
-
* - Data directory is modified at 12:30:00
|
|
59
|
-
* - We ask for modified files after 12:29:00 and got 3 files (1.json, 2.json and 3.json)
|
|
60
|
-
* - File "1.json" is processed and added to the store at 12:30:01
|
|
61
|
-
* - While "2.json" is being processed and added to the store, data directory is modified
|
|
62
|
-
* again at 12:30:02 and "1.json" is changed.
|
|
63
|
-
* - File "3.json" is processed and added to the store at 12:30:03
|
|
64
|
-
*
|
|
65
|
-
* At this moment, there are two options:
|
|
43
|
+
* Initial unique id when the store was initially loaded.
|
|
44
|
+
*/
|
|
45
|
+
initialUniqueId: string;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Current unique id to tell when was the store last synced.
|
|
66
49
|
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* - Data directory is modified again at 12:30:10 and only "3.json" is changed
|
|
70
|
-
* - We ask for modified files after 12:30:03 and get only "3.json"
|
|
71
|
-
* - Modification done at 12:30:02 to "1.json" is lost and cannot be recovered unless "1.json" is
|
|
72
|
-
* modified again, or Data Server is restarted
|
|
50
|
+
* Every time the store is updated, this id is set with the last modification
|
|
51
|
+
* unique id of the data directory, not the date when the store has finished updating.
|
|
73
52
|
*
|
|
74
|
-
* 2) Setting syncDate to the last modification date of the data directory:
|
|
75
|
-
* - Store syncDate is set to 12:30:00
|
|
76
|
-
* - Data directory is modified again at 12:30:10 and only "3.json" is changed
|
|
77
|
-
* - We ask for modified files after 12:30:00 and get "1.json" and "3.json"
|
|
78
|
-
* - Modification done at 12:30:02 to "1.json" is properly processed
|
|
79
53
|
*/
|
|
80
|
-
|
|
54
|
+
currentUniqueId: string;
|
|
81
55
|
|
|
82
56
|
/**
|
|
83
57
|
* Map that holds all data for all files.
|
|
@@ -105,7 +79,7 @@ export type Store = {
|
|
|
105
79
|
data: StoreData<string, any>;
|
|
106
80
|
|
|
107
81
|
/**
|
|
108
|
-
* A list of deleted files
|
|
82
|
+
* A list of deleted files since last reboot, to be able to track them.
|
|
109
83
|
*/
|
|
110
84
|
deleted: Set<string>;
|
|
111
85
|
|
|
@@ -119,7 +93,7 @@ export type Store = {
|
|
|
119
93
|
url: Map<string, any>;
|
|
120
94
|
|
|
121
95
|
/**
|
|
122
|
-
* An index that holds all data for all files, keyed by their
|
|
96
|
+
* An index that holds all data for all files, keyed by their langcode and UUID.
|
|
123
97
|
*/
|
|
124
98
|
uuid: Map<string, any>;
|
|
125
99
|
|
|
@@ -168,6 +142,11 @@ export type StoreSubsetOptions = {
|
|
|
168
142
|
* Object holding a subset of items from the store.
|
|
169
143
|
*/
|
|
170
144
|
export type StoreSubset = {
|
|
145
|
+
/**
|
|
146
|
+
* Map of elements, with the filename as key
|
|
147
|
+
*/
|
|
148
|
+
map: Map<string, any>;
|
|
149
|
+
|
|
171
150
|
/**
|
|
172
151
|
* Array of filenames in this subset.
|
|
173
152
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { config } from '@lib/config';
|
|
2
2
|
import { getFileContent } from '@lib/utils/fs';
|
|
3
|
-
import {
|
|
3
|
+
import { Json } from '@lib/utils/object/object.types';
|
|
4
4
|
import { StoreManager } from './store.types';
|
|
5
5
|
import { hookManager } from './hook';
|
|
6
6
|
import { store, resetStore } from '.';
|
|
@@ -16,7 +16,12 @@ import { dependencyTagger } from './dependency/dependencyTagger';
|
|
|
16
16
|
*
|
|
17
17
|
* @returns Object with two properties, "raw" and "json".
|
|
18
18
|
*/
|
|
19
|
-
const setFileIntoStore = (
|
|
19
|
+
const setFileIntoStore = (
|
|
20
|
+
relativeFilepath: string,
|
|
21
|
+
): {
|
|
22
|
+
fileContent: Json | string | null;
|
|
23
|
+
previousStoredData: Json | null;
|
|
24
|
+
} => {
|
|
20
25
|
const absoluteFilePath = `${config.dataDir}/${relativeFilepath}`;
|
|
21
26
|
let fileContent = getFileContent(absoluteFilePath);
|
|
22
27
|
|
|
@@ -26,10 +31,12 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
|
|
|
26
31
|
fileContent,
|
|
27
32
|
});
|
|
28
33
|
|
|
34
|
+
let previousStoredData: any = null;
|
|
35
|
+
|
|
29
36
|
const dataToStore = fileContent.json || fileContent.raw;
|
|
37
|
+
const previousData = store.data.get(relativeFilepath);
|
|
30
38
|
if (fileContent.json) {
|
|
31
39
|
// Check if the object already exists to make sure we don't break the reference
|
|
32
|
-
const previousData = store.data.get(relativeFilepath);
|
|
33
40
|
if (dataToStore && typeof dataToStore === 'object') {
|
|
34
41
|
if (previousData) {
|
|
35
42
|
// Delete include dependencies.
|
|
@@ -51,8 +58,10 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
|
|
|
51
58
|
store.index.uuid.get(langcode)?.delete(uuid);
|
|
52
59
|
}
|
|
53
60
|
|
|
54
|
-
// Delete all referenced object properties
|
|
61
|
+
// Delete all referenced object properties and copy them to another object.
|
|
62
|
+
previousStoredData = {};
|
|
55
63
|
Object.keys(previousData).forEach(key => {
|
|
64
|
+
previousStoredData[key] = previousData[key];
|
|
56
65
|
delete previousData[key];
|
|
57
66
|
});
|
|
58
67
|
// hydrate new object properties into referenced object
|
|
@@ -78,26 +87,31 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
|
|
|
78
87
|
}
|
|
79
88
|
|
|
80
89
|
// Add include dependencies.
|
|
81
|
-
dependencyIncludeHelper.
|
|
90
|
+
dependencyIncludeHelper.addIncludeDependencies(
|
|
82
91
|
relativeFilepath,
|
|
83
92
|
dataToStore,
|
|
84
93
|
);
|
|
85
94
|
}
|
|
95
|
+
} else {
|
|
96
|
+
previousStoredData = previousData;
|
|
86
97
|
}
|
|
87
98
|
store.data.set(relativeFilepath, dataToStore);
|
|
88
99
|
|
|
89
100
|
// Remove this path from store.deleted
|
|
90
101
|
store.deleted.delete(relativeFilepath);
|
|
91
102
|
|
|
92
|
-
return fileContent;
|
|
103
|
+
return { fileContent: dataToStore, previousStoredData };
|
|
93
104
|
};
|
|
94
105
|
|
|
95
106
|
export const storeManager: StoreManager = {
|
|
96
107
|
add: (relativeFilepath: string): StoreManager => {
|
|
97
|
-
const fileContent = setFileIntoStore(relativeFilepath);
|
|
108
|
+
const { fileContent } = setFileIntoStore(relativeFilepath);
|
|
98
109
|
|
|
99
110
|
// Invoke "store add" hook.
|
|
100
|
-
hookManager.invokeOnStoreItemAdd({
|
|
111
|
+
hookManager.invokeOnStoreItemAdd({
|
|
112
|
+
relativeFilepath,
|
|
113
|
+
storeItem: fileContent,
|
|
114
|
+
});
|
|
101
115
|
|
|
102
116
|
return storeManager;
|
|
103
117
|
},
|
|
@@ -107,11 +121,12 @@ export const storeManager: StoreManager = {
|
|
|
107
121
|
if (storedData) {
|
|
108
122
|
hookManager.invokeOnStoreItemBeforeUpdate({
|
|
109
123
|
relativeFilepath,
|
|
110
|
-
|
|
124
|
+
storeItem: storedData,
|
|
111
125
|
});
|
|
112
126
|
}
|
|
113
127
|
|
|
114
|
-
const fileContent =
|
|
128
|
+
const { fileContent, previousStoredData } =
|
|
129
|
+
setFileIntoStore(relativeFilepath);
|
|
115
130
|
|
|
116
131
|
// Invalidate this item.
|
|
117
132
|
dependencyTagger.invalidateTags([relativeFilepath]);
|
|
@@ -119,7 +134,8 @@ export const storeManager: StoreManager = {
|
|
|
119
134
|
// Invoke "store update" hook.
|
|
120
135
|
hookManager.invokeOnStoreItemAfterUpdate({
|
|
121
136
|
relativeFilepath,
|
|
122
|
-
fileContent,
|
|
137
|
+
storeItem: fileContent,
|
|
138
|
+
previousStoreItem: previousStoredData,
|
|
123
139
|
});
|
|
124
140
|
|
|
125
141
|
return storeManager;
|
|
@@ -154,7 +170,7 @@ export const storeManager: StoreManager = {
|
|
|
154
170
|
// Invoke "store remove" hook.
|
|
155
171
|
hookManager.invokeOnStoreItemDelete({
|
|
156
172
|
relativeFilepath,
|
|
157
|
-
|
|
173
|
+
storeItem: storedData,
|
|
158
174
|
});
|
|
159
175
|
|
|
160
176
|
return storeManager;
|
|
@@ -11,8 +11,13 @@ describe('workDirHelper test', () => {
|
|
|
11
11
|
|
|
12
12
|
it(`Gets modified and deleted files from static-suite log`, () => {
|
|
13
13
|
expect(
|
|
14
|
-
workDirHelper.
|
|
14
|
+
workDirHelper.getChangedFilesBetween(
|
|
15
|
+
'2021-08-16_07-11-55.666963__3503',
|
|
16
|
+
'2021-08-17_08-50-00.235800__1900',
|
|
17
|
+
),
|
|
15
18
|
).toEqual({
|
|
19
|
+
fromUniqueId: '2021-08-16_07-11-55.666963__3503',
|
|
20
|
+
toUniqueId: '2021-08-17_08-50-00.235800__1900',
|
|
16
21
|
deleted: ['es/entity/node/article/10.json'],
|
|
17
22
|
updated: [
|
|
18
23
|
'en/entity/node/article/10.json',
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { config } from '@lib/config';
|
|
4
|
-
import {
|
|
4
|
+
import { getChangedLinesBetween } from '../getChangedLinesBetween';
|
|
5
5
|
|
|
6
|
-
describe('workDir storage
|
|
7
|
-
describe('
|
|
6
|
+
describe('workDir storage getChangedLinesBetween test', () => {
|
|
7
|
+
describe('getChangedLinesBetween', () => {
|
|
8
8
|
config.workDir = fs.realpathSync(
|
|
9
9
|
path.resolve('src/__tests__/fixtures/work'),
|
|
10
10
|
);
|
|
11
11
|
|
|
12
12
|
it(`Gets lines of log since given date`, () => {
|
|
13
|
-
expect(
|
|
13
|
+
expect(
|
|
14
|
+
getChangedLinesBetween(
|
|
15
|
+
'2021-08-16_07-11-55.666963__3503',
|
|
16
|
+
'2021-08-17_08-50-00.235800__1900',
|
|
17
|
+
),
|
|
18
|
+
).toEqual([
|
|
14
19
|
'2021-08-16_07-11-55.666963__3504 delete [ID: 10] Prueba y cultiva tus propias hierbas | static-local://es/entity/node/article/10.json',
|
|
15
20
|
'2021-08-16_07-12-45.771095__5959 write [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
|
|
16
21
|
'2021-08-16_07-12-46.771095__5959 delete [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import { logger } from '@lib/utils/logger';
|
|
3
|
+
import { isUniqueId } from '@lib/utils/string';
|
|
4
|
+
import { getLogFile } from './getLogFile';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Get changed files between a set of dates.
|
|
8
|
+
*
|
|
9
|
+
* @param fromUniqueId - Unique id to search from.
|
|
10
|
+
* @param toUniqueId - Unique id to search to.
|
|
11
|
+
*
|
|
12
|
+
* @remarks
|
|
13
|
+
* It searches for lines grater than fromUniqueId and lower
|
|
14
|
+
* or equal to toUniqueId, to avoid getting repeated entries
|
|
15
|
+
* when dates are advanced in blocks (from 1 to 2, from 2 to 3,
|
|
16
|
+
* from 3 to 4, etc).
|
|
17
|
+
*
|
|
18
|
+
* @returns Array of changed lines
|
|
19
|
+
*/
|
|
20
|
+
export const getChangedLinesBetween = (
|
|
21
|
+
fromUniqueId: string,
|
|
22
|
+
toUniqueId: string,
|
|
23
|
+
): string[] => {
|
|
24
|
+
let allLines: string[] = [];
|
|
25
|
+
const logFile = getLogFile();
|
|
26
|
+
if (logFile) {
|
|
27
|
+
try {
|
|
28
|
+
allLines = fs.readFileSync(logFile).toString().trim().split('\n');
|
|
29
|
+
} catch (e) {
|
|
30
|
+
logger.error(
|
|
31
|
+
`Error reading metadata log file located at ${`logFile`}: ${e}`,
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const changedLines = allLines.filter(line => {
|
|
37
|
+
const uniqueId = line.substring(0, 32);
|
|
38
|
+
return (
|
|
39
|
+
uniqueId &&
|
|
40
|
+
isUniqueId(uniqueId) &&
|
|
41
|
+
uniqueId > fromUniqueId &&
|
|
42
|
+
uniqueId <= toUniqueId
|
|
43
|
+
);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
logger.debug(`Found ${changedLines.length} changed lines`);
|
|
47
|
+
changedLines.forEach(line => {
|
|
48
|
+
logger.debug(`Found changed line: ${line}`);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
return changedLines;
|
|
52
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { logger } from '@lib/utils/logger';
|
|
1
2
|
import { LogLineData } from '../workDir.types';
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -8,13 +9,31 @@ import { LogLineData } from '../workDir.types';
|
|
|
8
9
|
* @returns An object with parsed data, or null if line format is not valid.
|
|
9
10
|
*/
|
|
10
11
|
export const getDataFromLogLine = (line: string): LogLineData | null => {
|
|
11
|
-
const matches = line.match(/^(\S+) (\S+) \[ID: ([^\]]+)] (.+)
|
|
12
|
+
const matches = line.match(/^(\S+) (\S+) \[ID: ([^\]]+)] (.+)$/);
|
|
12
13
|
if (matches) {
|
|
13
14
|
const uniqueId = matches[1];
|
|
14
15
|
const operation = matches[2];
|
|
15
16
|
const fileId = matches[3];
|
|
16
|
-
|
|
17
|
-
const
|
|
17
|
+
// Some lines contain more than one "|"
|
|
18
|
+
const labelUriParts = matches[4].split(' | ');
|
|
19
|
+
const uri = labelUriParts.pop();
|
|
20
|
+
const fileLabel = labelUriParts.join(' | ');
|
|
21
|
+
const uriTarget = uri?.split('://').pop();
|
|
22
|
+
|
|
23
|
+
if (
|
|
24
|
+
!uniqueId ||
|
|
25
|
+
(operation !== 'write' && operation !== 'delete') ||
|
|
26
|
+
!fileId ||
|
|
27
|
+
!fileLabel ||
|
|
28
|
+
!uriTarget
|
|
29
|
+
) {
|
|
30
|
+
const errorMessage = `Line ${line} meets parsing criteria but it contains invalid data. Regex match: ${JSON.stringify(
|
|
31
|
+
matches,
|
|
32
|
+
)}`;
|
|
33
|
+
logger.error(errorMessage);
|
|
34
|
+
throw Error(errorMessage);
|
|
35
|
+
}
|
|
36
|
+
|
|
18
37
|
if (operation === 'write' || operation === 'delete') {
|
|
19
38
|
return {
|
|
20
39
|
uniqueId,
|
|
@@ -3,6 +3,6 @@ These functions are separated from workDirHelper, since they are specific
|
|
|
3
3
|
to the current storage backend (log files saved on disk) and they can change
|
|
4
4
|
in the future.
|
|
5
5
|
*/
|
|
6
|
-
export * from './
|
|
6
|
+
export * from './getChangedLinesBetween';
|
|
7
7
|
export * from './getDataFromLogLine';
|
|
8
8
|
export * from './getLogFile';
|
|
@@ -2,6 +2,16 @@
|
|
|
2
2
|
* A group of changed files in Static Suite's data dir.
|
|
3
3
|
*/
|
|
4
4
|
export type ChangedFiles = {
|
|
5
|
+
/**
|
|
6
|
+
* A unique id representing the date from which those changes are obtained.
|
|
7
|
+
*/
|
|
8
|
+
fromUniqueId: string;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* A unique id representing the date until which those changes are obtained.
|
|
12
|
+
*/
|
|
13
|
+
toUniqueId: string;
|
|
14
|
+
|
|
5
15
|
/**
|
|
6
16
|
* A list of updated files (both newly added and changed files).
|
|
7
17
|
*/
|
|
@@ -17,7 +27,7 @@ export type ChangedFiles = {
|
|
|
17
27
|
*/
|
|
18
28
|
export type LogLineData = {
|
|
19
29
|
/**
|
|
20
|
-
* The unique
|
|
30
|
+
* The unique id of the write/delete operation from Static Suite.
|
|
21
31
|
*/
|
|
22
32
|
uniqueId: string;
|
|
23
33
|
|
|
@@ -1,34 +1,63 @@
|
|
|
1
|
-
import
|
|
1
|
+
import fs from 'fs';
|
|
2
2
|
import { logger } from '@lib/utils/logger';
|
|
3
|
+
import { isUniqueId } from '@lib/utils/string';
|
|
3
4
|
import { ChangedFiles, LogLineData } from './workDir.types';
|
|
4
5
|
import {
|
|
5
6
|
getLogFile,
|
|
6
|
-
|
|
7
|
+
getChangedLinesBetween,
|
|
7
8
|
getDataFromLogLine,
|
|
8
9
|
} from './storage';
|
|
9
10
|
|
|
11
|
+
/**
|
|
12
|
+
* The unique id of the Unix Epoch (00:00:00 UTC on 1 January 1970)
|
|
13
|
+
*/
|
|
14
|
+
export const unixEpochUniqueId = '1970-01-01_00-00-00.000000__0000';
|
|
15
|
+
|
|
10
16
|
export const workDirHelper = {
|
|
11
17
|
/**
|
|
12
|
-
* Gets
|
|
18
|
+
* Gets unique id of last modification of work directory.
|
|
13
19
|
*
|
|
14
|
-
* @returns The
|
|
20
|
+
* @returns The unique id of last modification of work directory, or null if directory not found.
|
|
15
21
|
*/
|
|
16
|
-
|
|
22
|
+
getModificationUniqueId: (): string | null => {
|
|
23
|
+
let modificationUniqueId: string | null = null;
|
|
17
24
|
const logFile = getLogFile();
|
|
18
|
-
|
|
25
|
+
let allLines: string[] = [];
|
|
26
|
+
if (logFile) {
|
|
27
|
+
try {
|
|
28
|
+
allLines = fs.readFileSync(logFile).toString().trim().split('\n');
|
|
29
|
+
} catch (e) {
|
|
30
|
+
logger.error(
|
|
31
|
+
`Error reading metadata log file located at ${`logFile`}: ${e}`,
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
const lastLine = allLines.slice(-1)[0];
|
|
35
|
+
const lastLineUniqueId = lastLine.substring(0, 32);
|
|
36
|
+
if (isUniqueId(lastLineUniqueId)) {
|
|
37
|
+
modificationUniqueId = lastLineUniqueId;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return modificationUniqueId;
|
|
19
42
|
},
|
|
20
43
|
|
|
21
44
|
/**
|
|
22
45
|
* Get changed files since a date.
|
|
23
46
|
*
|
|
24
|
-
* @param
|
|
47
|
+
* @param fromUniqueId - Date to search from.
|
|
48
|
+
* @param toUniqueId - Date to search to.
|
|
25
49
|
*
|
|
26
|
-
* @returns Object with
|
|
50
|
+
* @returns Object with four properties:
|
|
27
51
|
* - updated: array of changed files.
|
|
28
52
|
* - deleted: array of deleted files.
|
|
53
|
+
* - fromTimestamp
|
|
54
|
+
* - toTimestamp
|
|
29
55
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
56
|
+
getChangedFilesBetween: (
|
|
57
|
+
fromUniqueId: string,
|
|
58
|
+
toUniqueId: string,
|
|
59
|
+
): ChangedFiles => {
|
|
60
|
+
const changedLines = getChangedLinesBetween(fromUniqueId, toUniqueId);
|
|
32
61
|
const lineData: Record<string, LogLineData> = {};
|
|
33
62
|
changedLines.forEach(line => {
|
|
34
63
|
const dataFromLogLine = getDataFromLogLine(line);
|
|
@@ -54,6 +83,11 @@ export const workDirHelper = {
|
|
|
54
83
|
logger.debug(`Found deleted file "${file}"`);
|
|
55
84
|
});
|
|
56
85
|
|
|
57
|
-
return {
|
|
86
|
+
return {
|
|
87
|
+
updated,
|
|
88
|
+
deleted,
|
|
89
|
+
fromUniqueId,
|
|
90
|
+
toUniqueId,
|
|
91
|
+
};
|
|
58
92
|
},
|
|
59
93
|
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
2
3
|
import fg, { Options as fastGlobOptions } from 'fast-glob';
|
|
3
4
|
import chokidar from 'chokidar';
|
|
4
5
|
import { logger } from '@lib/utils/logger';
|
|
@@ -99,6 +100,23 @@ export const getModificationDate = (filePath: string): Date | null => {
|
|
|
99
100
|
return modificationDate;
|
|
100
101
|
};
|
|
101
102
|
|
|
103
|
+
/**
|
|
104
|
+
* Removes empty directories upwards.
|
|
105
|
+
*
|
|
106
|
+
* @param dir - Path to a directory
|
|
107
|
+
*/
|
|
108
|
+
export const removeEmptyDirsUpwards = (dir: string): void => {
|
|
109
|
+
const isEmpty = fs.readdirSync(dir).length === 0;
|
|
110
|
+
if (isEmpty) {
|
|
111
|
+
try {
|
|
112
|
+
fs.rmdirSync(dir);
|
|
113
|
+
} catch (e) {
|
|
114
|
+
logger.debug(`Error deleting empty directory "${dir}": ${e}`);
|
|
115
|
+
}
|
|
116
|
+
removeEmptyDirsUpwards(path.dirname(dir));
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
|
|
102
120
|
/**
|
|
103
121
|
* Watches for changes on a set of paths and attach listener to them
|
|
104
122
|
*
|
|
@@ -82,7 +82,11 @@ export const moduleManager = {
|
|
|
82
82
|
throw e;
|
|
83
83
|
}
|
|
84
84
|
if (internalModuleCache[resolvedModulePath].onModuleLoad) {
|
|
85
|
-
internalModuleCache[resolvedModulePath].onModuleLoad({
|
|
85
|
+
internalModuleCache[resolvedModulePath].onModuleLoad({
|
|
86
|
+
config,
|
|
87
|
+
store,
|
|
88
|
+
logger,
|
|
89
|
+
});
|
|
86
90
|
}
|
|
87
91
|
logger.debug(`Module ${resolvedModulePath} successfully loaded.`);
|
|
88
92
|
return internalModuleCache[resolvedModulePath];
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { logger } from '@lib/utils/logger';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
isUniqueId,
|
|
4
|
+
parseJsonString,
|
|
5
|
+
parseUniqueId,
|
|
6
|
+
parseURLSearchParams,
|
|
7
|
+
} from '../stringUtils';
|
|
3
8
|
|
|
4
9
|
describe('String utils test', () => {
|
|
5
10
|
describe('parseJsonString', () => {
|
|
@@ -27,4 +32,28 @@ describe('String utils test', () => {
|
|
|
27
32
|
expect(parsedQuery).toEqual(expectedValue);
|
|
28
33
|
});
|
|
29
34
|
});
|
|
35
|
+
|
|
36
|
+
describe('parseUniqueId', () => {
|
|
37
|
+
it('parses a valid unique id and returns a date', () => {
|
|
38
|
+
const uniqueIdDate = parseUniqueId('2022-09-28_11-42-00.001712__7220');
|
|
39
|
+
expect(uniqueIdDate?.getTime()).toEqual(1664365320001);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('parses a non valid unique id and returns null', () => {
|
|
43
|
+
const uniqueIdDate = parseUniqueId('non valid unique id');
|
|
44
|
+
expect(uniqueIdDate).toEqual(null);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe('isUniqueId', () => {
|
|
49
|
+
it('returns true for a valid unique id', () => {
|
|
50
|
+
const result = isUniqueId('2022-09-28_11-42-00.001712__7220');
|
|
51
|
+
expect(result).toEqual(true);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('returns false for a non valid unique id', () => {
|
|
55
|
+
const result = isUniqueId('non valid unique id');
|
|
56
|
+
expect(result).toEqual(false);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
30
59
|
});
|
|
@@ -59,3 +59,35 @@ export const parseURLSearchParams = (
|
|
|
59
59
|
});
|
|
60
60
|
return obj;
|
|
61
61
|
};
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Parses a unique id from Static Suite and returns a Date.
|
|
65
|
+
*
|
|
66
|
+
* @param uniqueId - A unique id from Static Suite
|
|
67
|
+
*
|
|
68
|
+
* @returns A date from that unique id, or null if it can be parsed.
|
|
69
|
+
*/
|
|
70
|
+
export const parseUniqueId = (uniqueId: string): Date | null => {
|
|
71
|
+
// todo - Do not use dates and use timestamps in all cases
|
|
72
|
+
// to avoid having to fix offsets.
|
|
73
|
+
const date = new Date();
|
|
74
|
+
const dateOffset = -(date.getTimezoneOffset() * 60 * 1000);
|
|
75
|
+
|
|
76
|
+
const dateString = uniqueId.replace(
|
|
77
|
+
/(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/,
|
|
78
|
+
'$1-$2-$3T$4:$5:$6.$7',
|
|
79
|
+
);
|
|
80
|
+
const parsedDate = Date.parse(dateString);
|
|
81
|
+
if (!Number.isNaN(parsedDate)) {
|
|
82
|
+
return new Date(parsedDate + dateOffset);
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Checks that a string is a unique id.
|
|
89
|
+
*
|
|
90
|
+
* @param uniqueId - A unique id t be checked
|
|
91
|
+
*/
|
|
92
|
+
export const isUniqueId = (uniqueId: string): boolean =>
|
|
93
|
+
/(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/.test(uniqueId);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"queryRunner copy.d.ts","sourceRoot":"","sources":["../../../src/lib/query/queryRunner copy.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,WAAW,EAIZ,MAAM,eAAe,CAAC;AAkBvB,eAAO,MAAM,WAAW,EAAE,WAiEzB,CAAC"}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.queryRunner = void 0;
|
|
7
|
-
const microtime_1 = __importDefault(require("microtime"));
|
|
8
|
-
const config_1 = require("@lib/config");
|
|
9
|
-
const store_1 = require("@lib/store");
|
|
10
|
-
const logger_1 = require("@lib/utils/logger");
|
|
11
|
-
const cache_1 = require("@lib/utils/cache");
|
|
12
|
-
const queryManager_1 = require("./queryManager");
|
|
13
|
-
const query_types_1 = require("./query.types");
|
|
14
|
-
let count = 0;
|
|
15
|
-
/**
|
|
16
|
-
* Creates a QueryErrorResponse object ready to be returned to clients.
|
|
17
|
-
*
|
|
18
|
-
* @param message - Error message
|
|
19
|
-
*
|
|
20
|
-
* @returns A JSON object with an error key
|
|
21
|
-
*
|
|
22
|
-
* @internal
|
|
23
|
-
*/
|
|
24
|
-
const createErrorResponse = (message) => {
|
|
25
|
-
logger_1.logger.error(message);
|
|
26
|
-
return { error: message };
|
|
27
|
-
};
|
|
28
|
-
exports.queryRunner = {
|
|
29
|
-
run: (queryId, args) => {
|
|
30
|
-
if (!config_1.config.queryDir) {
|
|
31
|
-
return createErrorResponse('No query directory ("queryDir") configured');
|
|
32
|
-
}
|
|
33
|
-
if (!queryId) {
|
|
34
|
-
return createErrorResponse('No query ID received');
|
|
35
|
-
}
|
|
36
|
-
const queryModuleInfo = queryManager_1.queryManager.getModuleGroupInfo().get(queryId);
|
|
37
|
-
if (!queryModuleInfo) {
|
|
38
|
-
return createErrorResponse(`Query module for ID ${queryId} not found`);
|
|
39
|
-
}
|
|
40
|
-
count += 1;
|
|
41
|
-
const startDate = microtime_1.default.now();
|
|
42
|
-
const argsString = JSON.stringify(args);
|
|
43
|
-
// Implement a query cache.
|
|
44
|
-
const cacheId = `${queryId}--${argsString}`;
|
|
45
|
-
let isCacheMiss = false;
|
|
46
|
-
const queryCache = cache_1.cache.bin('query');
|
|
47
|
-
let queryResult = queryCache.get(cacheId);
|
|
48
|
-
if (!queryResult) {
|
|
49
|
-
try {
|
|
50
|
-
const queryModule = queryModuleInfo.getModule();
|
|
51
|
-
const queryResponse = queryModule.default({
|
|
52
|
-
store: store_1.store,
|
|
53
|
-
args,
|
|
54
|
-
});
|
|
55
|
-
queryResult = {
|
|
56
|
-
result: queryResponse?.result,
|
|
57
|
-
contentType: queryResponse?.contentType || 'application/json',
|
|
58
|
-
tags: queryResponse?.tags,
|
|
59
|
-
};
|
|
60
|
-
isCacheMiss = true;
|
|
61
|
-
if (queryResponse?.cacheable !== false) {
|
|
62
|
-
queryCache.set(cacheId, queryResult);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
catch (e) {
|
|
66
|
-
// Log error and rethrow.
|
|
67
|
-
logger_1.logger.error(`#${count} Query error: "${queryId}", args "${argsString}": ${e}`);
|
|
68
|
-
throw e;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
const execTimeMs = (microtime_1.default.now() - startDate) / 1000;
|
|
72
|
-
const response = {
|
|
73
|
-
data: queryResult.result,
|
|
74
|
-
metadata: {
|
|
75
|
-
contentType: queryResult?.contentType || 'application/json',
|
|
76
|
-
execTimeMs,
|
|
77
|
-
cache: isCacheMiss ? query_types_1.CacheStatus.MISS : query_types_1.CacheStatus.HIT,
|
|
78
|
-
tags: queryResult.tags,
|
|
79
|
-
},
|
|
80
|
-
};
|
|
81
|
-
/* logger.debug(
|
|
82
|
-
`#${count} Query "${queryId}" args "${argsString}" took ${execTimeMs} ms.`,
|
|
83
|
-
); */
|
|
84
|
-
return response;
|
|
85
|
-
},
|
|
86
|
-
getCount: () => count,
|
|
87
|
-
};
|