@static-suite/static-suite-data-server 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +564 -104
  2. package/dist/http/README.md +564 -104
  3. package/dist/http/routes/dependency.d.ts +10 -0
  4. package/dist/http/routes/dependency.d.ts.map +1 -0
  5. package/dist/http/routes/dependency.js +71 -0
  6. package/dist/http/routes/dump.d.ts.map +1 -1
  7. package/dist/http/routes/dump.js +15 -10
  8. package/dist/http/routes/home.js +2 -1
  9. package/dist/http/routes/index.d.ts.map +1 -1
  10. package/dist/http/routes/index.js +8 -0
  11. package/dist/http/routes/status.d.ts.map +1 -1
  12. package/dist/http/routes/status.js +1 -6
  13. package/dist/http/views/dependencyIndex.pug +14 -0
  14. package/dist/lib/dataServer.d.ts +2 -1
  15. package/dist/lib/dataServer.d.ts.map +1 -1
  16. package/dist/lib/query/query.types.d.ts +2 -2
  17. package/dist/lib/query/query.types.d.ts.map +1 -1
  18. package/dist/lib/query/query.types.js +2 -2
  19. package/dist/lib/query/queryRunner.d.ts.map +1 -1
  20. package/dist/lib/query/queryRunner.js +3 -1
  21. package/dist/lib/store/dataDir/dataDir.types.d.ts +5 -5
  22. package/dist/lib/store/dataDir/dataDir.types.d.ts.map +1 -1
  23. package/dist/lib/store/dataDir/dataDirManager.d.ts.map +1 -1
  24. package/dist/lib/store/dataDir/dataDirManager.js +23 -12
  25. package/dist/lib/store/dependency/dependency.types.d.ts.map +1 -1
  26. package/dist/lib/store/dependency/dependencyFileHelper.d.ts +1 -1
  27. package/dist/lib/store/dependency/dependencyFileHelper.d.ts.map +1 -1
  28. package/dist/lib/store/dependency/dependencyFileHelper.js +5 -3
  29. package/dist/lib/store/dependency/dependencyManager.d.ts +12 -0
  30. package/dist/lib/store/dependency/dependencyManager.d.ts.map +1 -1
  31. package/dist/lib/store/dependency/dependencyManager.js +7 -5
  32. package/dist/lib/store/dependency/dependencyTagger.d.ts.map +1 -1
  33. package/dist/lib/store/dependency/dependencyTagger.js +3 -38
  34. package/dist/lib/store/diff/{diffManager.types.d.ts → diff.types.d.ts} +14 -6
  35. package/dist/lib/store/diff/diff.types.d.ts.map +1 -0
  36. package/dist/lib/store/{dependency/dependencyManager.types.js → diff/diff.types.js} +0 -0
  37. package/dist/lib/store/diff/diffManager.d.ts +1 -1
  38. package/dist/lib/store/diff/diffManager.d.ts.map +1 -1
  39. package/dist/lib/store/diff/diffManager.js +38 -81
  40. package/dist/lib/store/dump/{dumpManager.types.d.ts → dump.types.d.ts} +21 -8
  41. package/dist/lib/store/dump/dump.types.d.ts.map +1 -0
  42. package/dist/lib/store/{diff/diffManager.types.js → dump/dump.types.js} +0 -0
  43. package/dist/lib/store/dump/dumpManager.d.ts +1 -1
  44. package/dist/lib/store/dump/dumpManager.d.ts.map +1 -1
  45. package/dist/lib/store/dump/dumpManager.js +31 -56
  46. package/dist/lib/store/dump/dumpMetadataHelper.d.ts +7 -0
  47. package/dist/lib/store/dump/dumpMetadataHelper.d.ts.map +1 -0
  48. package/dist/lib/store/dump/dumpMetadataHelper.js +61 -0
  49. package/dist/lib/store/hook/hook.types.d.ts +49 -23
  50. package/dist/lib/store/hook/hook.types.d.ts.map +1 -1
  51. package/dist/lib/store/hook/hookManager.d.ts.map +1 -1
  52. package/dist/lib/store/hook/hookManager.js +19 -8
  53. package/dist/lib/store/store.d.ts.map +1 -1
  54. package/dist/lib/store/store.js +9 -4
  55. package/dist/lib/store/store.types.d.ts +14 -37
  56. package/dist/lib/store/store.types.d.ts.map +1 -1
  57. package/dist/lib/store/storeManager.d.ts.map +1 -1
  58. package/dist/lib/store/storeManager.js +20 -10
  59. package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts +16 -0
  60. package/dist/lib/store/workDir/storage/getChangedLinesBetween.d.ts.map +1 -0
  61. package/dist/lib/store/workDir/storage/{getChangedLinesSince.js → getChangedLinesBetween.js} +22 -20
  62. package/dist/lib/store/workDir/storage/getDataFromLogLine.d.ts.map +1 -1
  63. package/dist/lib/store/workDir/storage/getDataFromLogLine.js +16 -3
  64. package/dist/lib/store/workDir/storage/index.d.ts +1 -1
  65. package/dist/lib/store/workDir/storage/index.d.ts.map +1 -1
  66. package/dist/lib/store/workDir/storage/index.js +1 -1
  67. package/dist/lib/store/workDir/workDir.types.d.ts +9 -1
  68. package/dist/lib/store/workDir/workDir.types.d.ts.map +1 -1
  69. package/dist/lib/store/workDir/workDirHelper.d.ts +13 -6
  70. package/dist/lib/store/workDir/workDirHelper.d.ts.map +1 -1
  71. package/dist/lib/store/workDir/workDirHelper.js +42 -11
  72. package/dist/lib/utils/fs/fsUtils.d.ts +6 -0
  73. package/dist/lib/utils/fs/fsUtils.d.ts.map +1 -1
  74. package/dist/lib/utils/fs/fsUtils.js +20 -1
  75. package/dist/lib/utils/module/moduleManager.d.ts.map +1 -1
  76. package/dist/lib/utils/module/moduleManager.js +5 -1
  77. package/dist/lib/utils/string/stringUtils.d.ts +14 -0
  78. package/dist/lib/utils/string/stringUtils.d.ts.map +1 -1
  79. package/dist/lib/utils/string/stringUtils.js +28 -1
  80. package/dist/static-suite-data-server.d.ts +19 -39
  81. package/package.json +3 -3
  82. package/src/http/routes/dependency.ts +91 -0
  83. package/src/http/routes/dump.ts +21 -10
  84. package/src/http/routes/home.ts +2 -2
  85. package/src/http/routes/index.ts +16 -0
  86. package/src/http/routes/status.ts +1 -9
  87. package/src/http/views/dependencyIndex.pug +14 -0
  88. package/src/lib/dataServer.ts +2 -0
  89. package/src/lib/query/query.types.ts +2 -2
  90. package/src/lib/query/queryRunner.ts +3 -1
  91. package/src/lib/store/dataDir/dataDir.types.ts +5 -5
  92. package/src/lib/store/dataDir/dataDirManager.ts +30 -14
  93. package/src/lib/store/dependency/dependency.types.ts +0 -5
  94. package/src/lib/store/dependency/dependencyFileHelper.ts +5 -3
  95. package/src/lib/store/dependency/dependencyManager.ts +6 -3
  96. package/src/lib/store/dependency/dependencyTagger.ts +3 -39
  97. package/src/lib/store/diff/{diffManager.types.ts → diff.types.ts} +14 -5
  98. package/src/lib/store/diff/diffManager.ts +45 -84
  99. package/src/lib/store/dump/{dumpManager.types.ts → dump.types.ts} +24 -7
  100. package/src/lib/store/dump/dumpManager.ts +40 -75
  101. package/src/lib/store/dump/dumpMetadataHelper.ts +73 -0
  102. package/src/lib/store/hook/hook.types.ts +58 -23
  103. package/src/lib/store/hook/hookManager.ts +24 -9
  104. package/src/lib/store/store.ts +9 -4
  105. package/src/lib/store/store.types.ts +16 -37
  106. package/src/lib/store/storeManager.ts +28 -12
  107. package/src/lib/store/workDir/__tests__/workDirHelper.test.ts +6 -1
  108. package/src/lib/store/workDir/storage/__tests__/{getChangedLinesSince.test.ts → getChangedLinesBetween.test.ts} +9 -4
  109. package/src/lib/store/workDir/storage/getChangedLinesBetween.ts +52 -0
  110. package/src/lib/store/workDir/storage/getDataFromLogLine.ts +22 -3
  111. package/src/lib/store/workDir/storage/index.ts +1 -1
  112. package/src/lib/store/workDir/workDir.types.ts +11 -1
  113. package/src/lib/store/workDir/workDirHelper.ts +45 -11
  114. package/src/lib/utils/fs/fsUtils.ts +18 -0
  115. package/src/lib/utils/module/moduleManager.ts +5 -1
  116. package/src/lib/utils/string/__tests__/stringUtils.test.ts +30 -1
  117. package/src/lib/utils/string/stringUtils.ts +32 -0
  118. package/dist/lib/query/queryRunner copy.d.ts +0 -3
  119. package/dist/lib/query/queryRunner copy.d.ts.map +0 -1
  120. package/dist/lib/query/queryRunner copy.js +0 -87
  121. package/dist/lib/store/dependency/dependencyManager copy.d.ts +0 -6
  122. package/dist/lib/store/dependency/dependencyManager copy.d.ts.map +0 -1
  123. package/dist/lib/store/dependency/dependencyManager copy.js +0 -91
  124. package/dist/lib/store/dependency/dependencyManager.types.d.ts +0 -48
  125. package/dist/lib/store/dependency/dependencyManager.types.d.ts.map +0 -1
  126. package/dist/lib/store/dependency/dependencyTagManager.d.ts +0 -6
  127. package/dist/lib/store/dependency/dependencyTagManager.d.ts.map +0 -1
  128. package/dist/lib/store/dependency/dependencyTagManager.js +0 -33
  129. package/dist/lib/store/diff/diffManager.types.d.ts.map +0 -1
  130. package/dist/lib/store/diff/tracker.d.ts +0 -3
  131. package/dist/lib/store/diff/tracker.d.ts.map +0 -1
  132. package/dist/lib/store/diff/tracker.js +0 -82
  133. package/dist/lib/store/diff/tracker.types.d.ts +0 -31
  134. package/dist/lib/store/diff/tracker.types.d.ts.map +0 -1
  135. package/dist/lib/store/diff/tracker.types.js +0 -2
  136. package/dist/lib/store/dump/dumpManager.types.d.ts.map +0 -1
  137. package/dist/lib/store/dump/dumpManager.types.js +0 -2
  138. package/dist/lib/store/include/includeIndex.d.ts +0 -6
  139. package/dist/lib/store/include/includeIndex.d.ts.map +0 -1
  140. package/dist/lib/store/include/includeIndex.js +0 -65
  141. package/dist/lib/store/include/includeIndex.types.d.ts +0 -29
  142. package/dist/lib/store/include/includeIndex.types.d.ts.map +0 -1
  143. package/dist/lib/store/include/includeIndex.types.js +0 -2
  144. package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts +0 -9
  145. package/dist/lib/store/workDir/storage/getChangedLinesSince.d.ts.map +0 -1
  146. package/src/lib/store/workDir/storage/getChangedLinesSince.ts +0 -47
@@ -40,44 +40,18 @@ export interface StoreData<K, V> extends Map<K, V> {
40
40
  */
41
41
  export type Store = {
42
42
  /**
43
- * Date to tell when was the store last synced.
44
- *
45
- * Every time the store is updated, syncDate is set with the last modification
46
- * date of the data directory, not the date when the store has finished updating.
47
- *
48
- * Given this scenario:
49
- * - Data directory modified at 12:30:00
50
- * - Syncing the store takes 3 seconds and finishes at 12:30:03
51
- *
52
- * ... syncDate will be 12:30:00, not 12:30:03.
53
- *
54
- * Doing it the other way round can cause data inconsistencies, because syncing files
55
- * to the store takes time, it is done sequentially, and in the meanwhile, other
56
- * modifications can happen:
57
- * - Store last syncDate is 12:29:00
58
- * - Data directory is modified at 12:30:00
59
- * - We ask for modified files after 12:29:00 and got 3 files (1.json, 2.json and 3.json)
60
- * - File "1.json" is processed and added to the store at 12:30:01
61
- * - While "2.json" is being processed and added to the store, data directory is modified
62
- * again at 12:30:02 and "1.json" is changed.
63
- * - File "3.json" is processed and added to the store at 12:30:03
64
- *
65
- * At this moment, there are two options:
43
+ * Initial unique id when the store was initially loaded.
44
+ */
45
+ initialUniqueId: string;
46
+
47
+ /**
48
+ * Current unique id to tell when was the store last synced.
66
49
  *
67
- * 1) Setting syncDate to the current date:
68
- * - Store syncDate is set to 12:30:03
69
- * - Data directory is modified again at 12:30:10 and only "3.json" is changed
70
- * - We ask for modified files after 12:30:03 and get only "3.json"
71
- * - Modification done at 12:30:02 to "1.json" is lost and cannot be recovered unless "1.json" is
72
- * modified again, or Data Server is restarted
50
+ * Every time the store is updated, this id is set with the last modification
51
+ * unique id of the data directory, not the date when the store has finished updating.
73
52
  *
74
- * 2) Setting syncDate to the last modification date of the data directory:
75
- * - Store syncDate is set to 12:30:00
76
- * - Data directory is modified again at 12:30:10 and only "3.json" is changed
77
- * - We ask for modified files after 12:30:00 and get "1.json" and "3.json"
78
- * - Modification done at 12:30:02 to "1.json" is properly processed
79
53
  */
80
- syncDate: Date | null;
54
+ currentUniqueId: string;
81
55
 
82
56
  /**
83
57
  * Map that holds all data for all files.
@@ -105,7 +79,7 @@ export type Store = {
105
79
  data: StoreData<string, any>;
106
80
 
107
81
  /**
108
- * A list of deleted files from store, to be able to track them.
82
+ * A list of deleted files since last reboot, to be able to track them.
109
83
  */
110
84
  deleted: Set<string>;
111
85
 
@@ -119,7 +93,7 @@ export type Store = {
119
93
  url: Map<string, any>;
120
94
 
121
95
  /**
122
- * An index that holds all data for all files, keyed by their lang and uuid.
96
+ * An index that holds all data for all files, keyed by their langcode and UUID.
123
97
  */
124
98
  uuid: Map<string, any>;
125
99
 
@@ -168,6 +142,11 @@ export type StoreSubsetOptions = {
168
142
  * Object holding a subset of items from the store.
169
143
  */
170
144
  export type StoreSubset = {
145
+ /**
146
+ * Map of elements, with the filename as key
147
+ */
148
+ map: Map<string, any>;
149
+
171
150
  /**
172
151
  * Array of filenames in this subset.
173
152
  */
@@ -1,6 +1,6 @@
1
1
  import { config } from '@lib/config';
2
2
  import { getFileContent } from '@lib/utils/fs';
3
- import { FileType } from '@lib/utils/fs/fs.types';
3
+ import { Json } from '@lib/utils/object/object.types';
4
4
  import { StoreManager } from './store.types';
5
5
  import { hookManager } from './hook';
6
6
  import { store, resetStore } from '.';
@@ -16,7 +16,12 @@ import { dependencyTagger } from './dependency/dependencyTagger';
16
16
  *
17
17
  * @returns Object with two properties, "raw" and "json".
18
18
  */
19
- const setFileIntoStore = (relativeFilepath: string): FileType => {
19
+ const setFileIntoStore = (
20
+ relativeFilepath: string,
21
+ ): {
22
+ fileContent: Json | string | null;
23
+ previousStoredData: Json | null;
24
+ } => {
20
25
  const absoluteFilePath = `${config.dataDir}/${relativeFilepath}`;
21
26
  let fileContent = getFileContent(absoluteFilePath);
22
27
 
@@ -26,10 +31,12 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
26
31
  fileContent,
27
32
  });
28
33
 
34
+ let previousStoredData: any = null;
35
+
29
36
  const dataToStore = fileContent.json || fileContent.raw;
37
+ const previousData = store.data.get(relativeFilepath);
30
38
  if (fileContent.json) {
31
39
  // Check if the object already exists to make sure we don't break the reference
32
- const previousData = store.data.get(relativeFilepath);
33
40
  if (dataToStore && typeof dataToStore === 'object') {
34
41
  if (previousData) {
35
42
  // Delete include dependencies.
@@ -51,8 +58,10 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
51
58
  store.index.uuid.get(langcode)?.delete(uuid);
52
59
  }
53
60
 
54
- // Delete all referenced object properties
61
+ // Delete all referenced object properties and copy them to another object.
62
+ previousStoredData = {};
55
63
  Object.keys(previousData).forEach(key => {
64
+ previousStoredData[key] = previousData[key];
56
65
  delete previousData[key];
57
66
  });
58
67
  // hydrate new object properties into referenced object
@@ -78,26 +87,31 @@ const setFileIntoStore = (relativeFilepath: string): FileType => {
78
87
  }
79
88
 
80
89
  // Add include dependencies.
81
- dependencyIncludeHelper.setIncludeDependencies(
90
+ dependencyIncludeHelper.addIncludeDependencies(
82
91
  relativeFilepath,
83
92
  dataToStore,
84
93
  );
85
94
  }
95
+ } else {
96
+ previousStoredData = previousData;
86
97
  }
87
98
  store.data.set(relativeFilepath, dataToStore);
88
99
 
89
100
  // Remove this path from store.deleted
90
101
  store.deleted.delete(relativeFilepath);
91
102
 
92
- return fileContent;
103
+ return { fileContent: dataToStore, previousStoredData };
93
104
  };
94
105
 
95
106
  export const storeManager: StoreManager = {
96
107
  add: (relativeFilepath: string): StoreManager => {
97
- const fileContent = setFileIntoStore(relativeFilepath);
108
+ const { fileContent } = setFileIntoStore(relativeFilepath);
98
109
 
99
110
  // Invoke "store add" hook.
100
- hookManager.invokeOnStoreItemAdd({ relativeFilepath, fileContent });
111
+ hookManager.invokeOnStoreItemAdd({
112
+ relativeFilepath,
113
+ storeItem: fileContent,
114
+ });
101
115
 
102
116
  return storeManager;
103
117
  },
@@ -107,11 +121,12 @@ export const storeManager: StoreManager = {
107
121
  if (storedData) {
108
122
  hookManager.invokeOnStoreItemBeforeUpdate({
109
123
  relativeFilepath,
110
- fileContent: storedData,
124
+ storeItem: storedData,
111
125
  });
112
126
  }
113
127
 
114
- const fileContent = setFileIntoStore(relativeFilepath);
128
+ const { fileContent, previousStoredData } =
129
+ setFileIntoStore(relativeFilepath);
115
130
 
116
131
  // Invalidate this item.
117
132
  dependencyTagger.invalidateTags([relativeFilepath]);
@@ -119,7 +134,8 @@ export const storeManager: StoreManager = {
119
134
  // Invoke "store update" hook.
120
135
  hookManager.invokeOnStoreItemAfterUpdate({
121
136
  relativeFilepath,
122
- fileContent,
137
+ storeItem: fileContent,
138
+ previousStoreItem: previousStoredData,
123
139
  });
124
140
 
125
141
  return storeManager;
@@ -154,7 +170,7 @@ export const storeManager: StoreManager = {
154
170
  // Invoke "store remove" hook.
155
171
  hookManager.invokeOnStoreItemDelete({
156
172
  relativeFilepath,
157
- fileContent: storedData,
173
+ storeItem: storedData,
158
174
  });
159
175
 
160
176
  return storeManager;
@@ -11,8 +11,13 @@ describe('workDirHelper test', () => {
11
11
 
12
12
  it(`Gets modified and deleted files from static-suite log`, () => {
13
13
  expect(
14
- workDirHelper.getChangedFilesSince(new Date('2021-08-01')),
14
+ workDirHelper.getChangedFilesBetween(
15
+ '2021-08-16_07-11-55.666963__3503',
16
+ '2021-08-17_08-50-00.235800__1900',
17
+ ),
15
18
  ).toEqual({
19
+ fromUniqueId: '2021-08-16_07-11-55.666963__3503',
20
+ toUniqueId: '2021-08-17_08-50-00.235800__1900',
16
21
  deleted: ['es/entity/node/article/10.json'],
17
22
  updated: [
18
23
  'en/entity/node/article/10.json',
@@ -1,16 +1,21 @@
1
1
  import fs from 'fs';
2
2
  import path from 'path';
3
3
  import { config } from '@lib/config';
4
- import { getChangedLinesSince } from '../getChangedLinesSince';
4
+ import { getChangedLinesBetween } from '../getChangedLinesBetween';
5
5
 
6
- describe('workDir storage getChangedLinesSince test', () => {
7
- describe('getChangedLinesSince', () => {
6
+ describe('workDir storage getChangedLinesBetween test', () => {
7
+ describe('getChangedLinesBetween', () => {
8
8
  config.workDir = fs.realpathSync(
9
9
  path.resolve('src/__tests__/fixtures/work'),
10
10
  );
11
11
 
12
12
  it(`Gets lines of log since given date`, () => {
13
- expect(getChangedLinesSince(new Date('2021-08-01'))).toEqual([
13
+ expect(
14
+ getChangedLinesBetween(
15
+ '2021-08-16_07-11-55.666963__3503',
16
+ '2021-08-17_08-50-00.235800__1900',
17
+ ),
18
+ ).toEqual([
14
19
  '2021-08-16_07-11-55.666963__3504 delete [ID: 10] Prueba y cultiva tus propias hierbas | static-local://es/entity/node/article/10.json',
15
20
  '2021-08-16_07-12-45.771095__5959 write [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
16
21
  '2021-08-16_07-12-46.771095__5959 delete [ID: 10] Give it a go and grow your own herbs | static-local://en/entity/node/article/10.json',
@@ -0,0 +1,52 @@
1
+ import fs from 'fs';
2
+ import { logger } from '@lib/utils/logger';
3
+ import { isUniqueId } from '@lib/utils/string';
4
+ import { getLogFile } from './getLogFile';
5
+
6
+ /**
7
+ * Get changed files between a set of dates.
8
+ *
9
+ * @param fromUniqueId - Unique id to search from.
10
+ * @param toUniqueId - Unique id to search to.
11
+ *
12
+ * @remarks
13
+ * It searches for lines grater than fromUniqueId and lower
14
+ * or equal to toUniqueId, to avoid getting repeated entries
15
+ * when dates are advanced in blocks (from 1 to 2, from 2 to 3,
16
+ * from 3 to 4, etc).
17
+ *
18
+ * @returns Array of changed lines
19
+ */
20
+ export const getChangedLinesBetween = (
21
+ fromUniqueId: string,
22
+ toUniqueId: string,
23
+ ): string[] => {
24
+ let allLines: string[] = [];
25
+ const logFile = getLogFile();
26
+ if (logFile) {
27
+ try {
28
+ allLines = fs.readFileSync(logFile).toString().trim().split('\n');
29
+ } catch (e) {
30
+ logger.error(
31
+ `Error reading metadata log file located at ${`logFile`}: ${e}`,
32
+ );
33
+ }
34
+ }
35
+
36
+ const changedLines = allLines.filter(line => {
37
+ const uniqueId = line.substring(0, 32);
38
+ return (
39
+ uniqueId &&
40
+ isUniqueId(uniqueId) &&
41
+ uniqueId > fromUniqueId &&
42
+ uniqueId <= toUniqueId
43
+ );
44
+ });
45
+
46
+ logger.debug(`Found ${changedLines.length} changed lines`);
47
+ changedLines.forEach(line => {
48
+ logger.debug(`Found changed line: ${line}`);
49
+ });
50
+
51
+ return changedLines;
52
+ };
@@ -1,3 +1,4 @@
1
+ import { logger } from '@lib/utils/logger';
1
2
  import { LogLineData } from '../workDir.types';
2
3
 
3
4
  /**
@@ -8,13 +9,31 @@ import { LogLineData } from '../workDir.types';
8
9
  * @returns An object with parsed data, or null if line format is not valid.
9
10
  */
10
11
  export const getDataFromLogLine = (line: string): LogLineData | null => {
11
- const matches = line.match(/^(\S+) (\S+) \[ID: ([^\]]+)] (.+)/);
12
+ const matches = line.match(/^(\S+) (\S+) \[ID: ([^\]]+)] (.+)$/);
12
13
  if (matches) {
13
14
  const uniqueId = matches[1];
14
15
  const operation = matches[2];
15
16
  const fileId = matches[3];
16
- const [fileLabel, uri] = matches[4].split(' | ');
17
- const [, uriTarget] = uri.split('://');
17
+ // Some lines contain more than one "|"
18
+ const labelUriParts = matches[4].split(' | ');
19
+ const uri = labelUriParts.pop();
20
+ const fileLabel = labelUriParts.join(' | ');
21
+ const uriTarget = uri?.split('://').pop();
22
+
23
+ if (
24
+ !uniqueId ||
25
+ (operation !== 'write' && operation !== 'delete') ||
26
+ !fileId ||
27
+ !fileLabel ||
28
+ !uriTarget
29
+ ) {
30
+ const errorMessage = `Line ${line} meets parsing criteria but it contains invalid data. Regex match: ${JSON.stringify(
31
+ matches,
32
+ )}`;
33
+ logger.error(errorMessage);
34
+ throw Error(errorMessage);
35
+ }
36
+
18
37
  if (operation === 'write' || operation === 'delete') {
19
38
  return {
20
39
  uniqueId,
@@ -3,6 +3,6 @@ These functions are separated from workDirHelper, since they are specific
3
3
  to the current storage backend (log files saved on disk) and they can change
4
4
  in the future.
5
5
  */
6
- export * from './getChangedLinesSince';
6
+ export * from './getChangedLinesBetween';
7
7
  export * from './getDataFromLogLine';
8
8
  export * from './getLogFile';
@@ -2,6 +2,16 @@
2
2
  * A group of changed files in Static Suite's data dir.
3
3
  */
4
4
  export type ChangedFiles = {
5
+ /**
6
+ * A unique id representing the date from which those changes are obtained.
7
+ */
8
+ fromUniqueId: string;
9
+
10
+ /**
11
+ * A unique id representing the date until which those changes are obtained.
12
+ */
13
+ toUniqueId: string;
14
+
5
15
  /**
6
16
  * A list of updated files (both newly added and changed files).
7
17
  */
@@ -17,7 +27,7 @@ export type ChangedFiles = {
17
27
  */
18
28
  export type LogLineData = {
19
29
  /**
20
- * The unique ID of the write/delete operation from Static Suite.
30
+ * The unique id of the write/delete operation from Static Suite.
21
31
  */
22
32
  uniqueId: string;
23
33
 
@@ -1,34 +1,63 @@
1
- import { getModificationDate } from '@lib/utils/fs';
1
+ import fs from 'fs';
2
2
  import { logger } from '@lib/utils/logger';
3
+ import { isUniqueId } from '@lib/utils/string';
3
4
  import { ChangedFiles, LogLineData } from './workDir.types';
4
5
  import {
5
6
  getLogFile,
6
- getChangedLinesSince,
7
+ getChangedLinesBetween,
7
8
  getDataFromLogLine,
8
9
  } from './storage';
9
10
 
11
+ /**
12
+ * The unique id of the Unix Epoch (00:00:00 UTC on 1 January 1970)
13
+ */
14
+ export const unixEpochUniqueId = '1970-01-01_00-00-00.000000__0000';
15
+
10
16
  export const workDirHelper = {
11
17
  /**
12
- * Gets date of last modification of work directory.
18
+ * Gets unique id of last modification of work directory.
13
19
  *
14
- * @returns The date of last modification of work directory, or null if directory not found.
20
+ * @returns The unique id of last modification of work directory, or null if directory not found.
15
21
  */
16
- getModificationDate: (): Date | null => {
22
+ getModificationUniqueId: (): string | null => {
23
+ let modificationUniqueId: string | null = null;
17
24
  const logFile = getLogFile();
18
- return logFile ? getModificationDate(logFile) : null;
25
+ let allLines: string[] = [];
26
+ if (logFile) {
27
+ try {
28
+ allLines = fs.readFileSync(logFile).toString().trim().split('\n');
29
+ } catch (e) {
30
+ logger.error(
31
+ `Error reading metadata log file located at ${`logFile`}: ${e}`,
32
+ );
33
+ }
34
+ const lastLine = allLines.slice(-1)[0];
35
+ const lastLineUniqueId = lastLine.substring(0, 32);
36
+ if (isUniqueId(lastLineUniqueId)) {
37
+ modificationUniqueId = lastLineUniqueId;
38
+ }
39
+ }
40
+
41
+ return modificationUniqueId;
19
42
  },
20
43
 
21
44
  /**
22
45
  * Get changed files since a date.
23
46
  *
24
- * @param sinceDate - Date to search
47
+ * @param fromUniqueId - Date to search from.
48
+ * @param toUniqueId - Date to search to.
25
49
  *
26
- * @returns Object with two properties:
50
+ * @returns Object with four properties:
27
51
  * - updated: array of changed files.
28
52
  * - deleted: array of deleted files.
53
+ * - fromTimestamp
54
+ * - toTimestamp
29
55
  */
30
- getChangedFilesSince: (sinceDate: Date): ChangedFiles => {
31
- const changedLines = getChangedLinesSince(sinceDate);
56
+ getChangedFilesBetween: (
57
+ fromUniqueId: string,
58
+ toUniqueId: string,
59
+ ): ChangedFiles => {
60
+ const changedLines = getChangedLinesBetween(fromUniqueId, toUniqueId);
32
61
  const lineData: Record<string, LogLineData> = {};
33
62
  changedLines.forEach(line => {
34
63
  const dataFromLogLine = getDataFromLogLine(line);
@@ -54,6 +83,11 @@ export const workDirHelper = {
54
83
  logger.debug(`Found deleted file "${file}"`);
55
84
  });
56
85
 
57
- return { updated, deleted };
86
+ return {
87
+ updated,
88
+ deleted,
89
+ fromUniqueId,
90
+ toUniqueId,
91
+ };
58
92
  },
59
93
  };
@@ -1,4 +1,5 @@
1
1
  import fs from 'fs';
2
+ import path from 'path';
2
3
  import fg, { Options as fastGlobOptions } from 'fast-glob';
3
4
  import chokidar from 'chokidar';
4
5
  import { logger } from '@lib/utils/logger';
@@ -99,6 +100,23 @@ export const getModificationDate = (filePath: string): Date | null => {
99
100
  return modificationDate;
100
101
  };
101
102
 
103
+ /**
104
+ * Removes empty directories upwards.
105
+ *
106
+ * @param dir - Path to a directory
107
+ */
108
+ export const removeEmptyDirsUpwards = (dir: string): void => {
109
+ const isEmpty = fs.readdirSync(dir).length === 0;
110
+ if (isEmpty) {
111
+ try {
112
+ fs.rmdirSync(dir);
113
+ } catch (e) {
114
+ logger.debug(`Error deleting empty directory "${dir}": ${e}`);
115
+ }
116
+ removeEmptyDirsUpwards(path.dirname(dir));
117
+ }
118
+ };
119
+
102
120
  /**
103
121
  * Watches for changes on a set of paths and attach listener to them
104
122
  *
@@ -82,7 +82,11 @@ export const moduleManager = {
82
82
  throw e;
83
83
  }
84
84
  if (internalModuleCache[resolvedModulePath].onModuleLoad) {
85
- internalModuleCache[resolvedModulePath].onModuleLoad({ config, store });
85
+ internalModuleCache[resolvedModulePath].onModuleLoad({
86
+ config,
87
+ store,
88
+ logger,
89
+ });
86
90
  }
87
91
  logger.debug(`Module ${resolvedModulePath} successfully loaded.`);
88
92
  return internalModuleCache[resolvedModulePath];
@@ -1,5 +1,10 @@
1
1
  import { logger } from '@lib/utils/logger';
2
- import { parseJsonString, parseURLSearchParams } from '../stringUtils';
2
+ import {
3
+ isUniqueId,
4
+ parseJsonString,
5
+ parseUniqueId,
6
+ parseURLSearchParams,
7
+ } from '../stringUtils';
3
8
 
4
9
  describe('String utils test', () => {
5
10
  describe('parseJsonString', () => {
@@ -27,4 +32,28 @@ describe('String utils test', () => {
27
32
  expect(parsedQuery).toEqual(expectedValue);
28
33
  });
29
34
  });
35
+
36
+ describe('parseUniqueId', () => {
37
+ it('parses a valid unique id and returns a date', () => {
38
+ const uniqueIdDate = parseUniqueId('2022-09-28_11-42-00.001712__7220');
39
+ expect(uniqueIdDate?.getTime()).toEqual(1664365320001);
40
+ });
41
+
42
+ it('parses a non valid unique id and returns null', () => {
43
+ const uniqueIdDate = parseUniqueId('non valid unique id');
44
+ expect(uniqueIdDate).toEqual(null);
45
+ });
46
+ });
47
+
48
+ describe('isUniqueId', () => {
49
+ it('returns true for a valid unique id', () => {
50
+ const result = isUniqueId('2022-09-28_11-42-00.001712__7220');
51
+ expect(result).toEqual(true);
52
+ });
53
+
54
+ it('returns false for a non valid unique id', () => {
55
+ const result = isUniqueId('non valid unique id');
56
+ expect(result).toEqual(false);
57
+ });
58
+ });
30
59
  });
@@ -59,3 +59,35 @@ export const parseURLSearchParams = (
59
59
  });
60
60
  return obj;
61
61
  };
62
+
63
+ /**
64
+ * Parses a unique id from Static Suite and returns a Date.
65
+ *
66
+ * @param uniqueId - A unique id from Static Suite
67
+ *
68
+ * @returns A date from that unique id, or null if it can be parsed.
69
+ */
70
+ export const parseUniqueId = (uniqueId: string): Date | null => {
71
+ // todo - Do not use dates and use timestamps in all cases
72
+ // to avoid having to fix offsets.
73
+ const date = new Date();
74
+ const dateOffset = -(date.getTimezoneOffset() * 60 * 1000);
75
+
76
+ const dateString = uniqueId.replace(
77
+ /(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/,
78
+ '$1-$2-$3T$4:$5:$6.$7',
79
+ );
80
+ const parsedDate = Date.parse(dateString);
81
+ if (!Number.isNaN(parsedDate)) {
82
+ return new Date(parsedDate + dateOffset);
83
+ }
84
+ return null;
85
+ };
86
+
87
+ /**
88
+ * Checks that a string is a unique id.
89
+ *
90
+ * @param uniqueId - A unique id t be checked
91
+ */
92
+ export const isUniqueId = (uniqueId: string): boolean =>
93
+ /(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})\.(\d{3}).*/.test(uniqueId);
@@ -1,3 +0,0 @@
1
- import { QueryRunner } from './query.types';
2
- export declare const queryRunner: QueryRunner;
3
- //# sourceMappingURL=queryRunner%20copy.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"queryRunner copy.d.ts","sourceRoot":"","sources":["../../../src/lib/query/queryRunner copy.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,WAAW,EAIZ,MAAM,eAAe,CAAC;AAkBvB,eAAO,MAAM,WAAW,EAAE,WAiEzB,CAAC"}
@@ -1,87 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.queryRunner = void 0;
7
- const microtime_1 = __importDefault(require("microtime"));
8
- const config_1 = require("@lib/config");
9
- const store_1 = require("@lib/store");
10
- const logger_1 = require("@lib/utils/logger");
11
- const cache_1 = require("@lib/utils/cache");
12
- const queryManager_1 = require("./queryManager");
13
- const query_types_1 = require("./query.types");
14
- let count = 0;
15
- /**
16
- * Creates a QueryErrorResponse object ready to be returned to clients.
17
- *
18
- * @param message - Error message
19
- *
20
- * @returns A JSON object with an error key
21
- *
22
- * @internal
23
- */
24
- const createErrorResponse = (message) => {
25
- logger_1.logger.error(message);
26
- return { error: message };
27
- };
28
- exports.queryRunner = {
29
- run: (queryId, args) => {
30
- if (!config_1.config.queryDir) {
31
- return createErrorResponse('No query directory ("queryDir") configured');
32
- }
33
- if (!queryId) {
34
- return createErrorResponse('No query ID received');
35
- }
36
- const queryModuleInfo = queryManager_1.queryManager.getModuleGroupInfo().get(queryId);
37
- if (!queryModuleInfo) {
38
- return createErrorResponse(`Query module for ID ${queryId} not found`);
39
- }
40
- count += 1;
41
- const startDate = microtime_1.default.now();
42
- const argsString = JSON.stringify(args);
43
- // Implement a query cache.
44
- const cacheId = `${queryId}--${argsString}`;
45
- let isCacheMiss = false;
46
- const queryCache = cache_1.cache.bin('query');
47
- let queryResult = queryCache.get(cacheId);
48
- if (!queryResult) {
49
- try {
50
- const queryModule = queryModuleInfo.getModule();
51
- const queryResponse = queryModule.default({
52
- store: store_1.store,
53
- args,
54
- });
55
- queryResult = {
56
- result: queryResponse?.result,
57
- contentType: queryResponse?.contentType || 'application/json',
58
- tags: queryResponse?.tags,
59
- };
60
- isCacheMiss = true;
61
- if (queryResponse?.cacheable !== false) {
62
- queryCache.set(cacheId, queryResult);
63
- }
64
- }
65
- catch (e) {
66
- // Log error and rethrow.
67
- logger_1.logger.error(`#${count} Query error: "${queryId}", args "${argsString}": ${e}`);
68
- throw e;
69
- }
70
- }
71
- const execTimeMs = (microtime_1.default.now() - startDate) / 1000;
72
- const response = {
73
- data: queryResult.result,
74
- metadata: {
75
- contentType: queryResult?.contentType || 'application/json',
76
- execTimeMs,
77
- cache: isCacheMiss ? query_types_1.CacheStatus.MISS : query_types_1.CacheStatus.HIT,
78
- tags: queryResult.tags,
79
- },
80
- };
81
- /* logger.debug(
82
- `#${count} Query "${queryId}" args "${argsString}" took ${execTimeMs} ms.`,
83
- ); */
84
- return response;
85
- },
86
- getCount: () => count,
87
- };