data-primals-engine 1.7.2 → 1.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +160 -160
  2. package/client/package-lock.json +8430 -8430
  3. package/client/src/APIInfo.jsx +1 -1
  4. package/client/src/App.jsx +2 -1
  5. package/client/src/App.scss +1635 -1626
  6. package/client/src/AssistantChat.jsx +1 -0
  7. package/client/src/CalendarView.jsx +1 -0
  8. package/client/src/ContentView.jsx +3 -3
  9. package/client/src/Dashboard.jsx +5 -2
  10. package/client/src/DashboardChart.jsx +1 -0
  11. package/client/src/DashboardFlexViewItem.jsx +1 -0
  12. package/client/src/DashboardHtmlViewItem.jsx +1 -0
  13. package/client/src/DashboardView.jsx +2 -0
  14. package/client/src/DataEditor.jsx +0 -1
  15. package/client/src/DataImporter.jsx +489 -468
  16. package/client/src/DataLayout.jsx +6 -3
  17. package/client/src/DataTable.jsx +4 -3
  18. package/client/src/Dialog.jsx +92 -90
  19. package/client/src/Dialog.scss +122 -116
  20. package/client/src/DisplayFlexNodeRenderer.jsx +1 -0
  21. package/client/src/FlexBuilderControls.jsx +1 -0
  22. package/client/src/FlexBuilderPreview.jsx +1 -1
  23. package/client/src/FlexNode.jsx +1 -1
  24. package/client/src/HistoryDialog.jsx +3 -2
  25. package/client/src/KPIWidget.jsx +1 -1
  26. package/client/src/KanbanView.jsx +1 -0
  27. package/client/src/ModelCreator.jsx +4 -0
  28. package/client/src/ModelList.jsx +1 -0
  29. package/client/src/PackGallery.jsx +5 -4
  30. package/client/src/RTETrans.jsx +1 -1
  31. package/client/src/RelationField.jsx +2 -0
  32. package/client/src/RelationSelectorWidget.jsx +2 -0
  33. package/client/src/RelationValue.jsx +1 -0
  34. package/client/src/RestoreDialog.jsx +1 -0
  35. package/client/src/WorkflowEditor.jsx +3 -0
  36. package/client/src/contexts/CommandContext.jsx +2 -1
  37. package/client/src/contexts/ModelContext.jsx +3 -3
  38. package/client/src/hooks/data.js +1 -0
  39. package/client/src/hooks/useValidation.js +1 -0
  40. package/client/src/translations.js +24 -24
  41. package/doc/AI-assistance.md +87 -63
  42. package/doc/Concepts.md +122 -0
  43. package/doc/Custom-Endpoints.md +31 -0
  44. package/doc/Event-system.md +13 -14
  45. package/doc/Home.md +33 -0
  46. package/doc/Modules.md +83 -0
  47. package/doc/Packs-gallery.md +8 -23
  48. package/doc/Workflows.md +32 -0
  49. package/doc/automation-workflows.md +141 -102
  50. package/doc/dashboards-kpis-charts.md +47 -49
  51. package/doc/data-management.md +126 -120
  52. package/doc/data-models.md +68 -75
  53. package/doc/roles-permissions.md +144 -43
  54. package/doc/sharding-replication.md +158 -0
  55. package/doc/users.md +54 -30
  56. package/package.json +1 -1
  57. package/server.js +37 -37
  58. package/src/constants.js +7 -8
  59. package/src/data.js +521 -520
  60. package/src/email.js +157 -154
  61. package/src/engine.js +117 -7
  62. package/src/gameObject.js +6 -0
  63. package/src/i18n.js +0 -1
  64. package/src/modules/auth-google/index.js +53 -50
  65. package/src/modules/bucket.js +346 -339
  66. package/src/modules/data/data.backup.js +400 -376
  67. package/src/modules/data/data.cluster.js +410 -42
  68. package/src/modules/data/data.operations.js +3666 -3635
  69. package/src/modules/data/data.replication.js +106 -64
  70. package/src/modules/data/data.routes.js +87 -64
  71. package/src/modules/data/data.scheduling.js +2 -1
  72. package/src/modules/file.js +248 -247
  73. package/src/modules/user.js +11 -1
  74. package/src/sso.js +91 -25
  75. package/test/cluster.test.js +221 -0
  76. package/test/core.test.js +0 -2
  77. package/test/replication.test.js +163 -0
  78. package/doc/core-concepts.md +0 -33
@@ -1,377 +1,401 @@
1
- import {dbUrl, MongoClient, MongoDatabase} from "../../engine.js";
2
- import {getUserId, isDemoUser} from "../../data.js";
3
- import {Config} from "../../config.js";
4
- import {Event} from "../../events.js";
5
- import {getObjectHash} from "../../core.js";
6
- import path from "node:path";
7
- import {downloadFromS3, getUserS3Config, listS3Backups, uploadToS3} from "../bucket.js";
8
- import fs from "node:fs";
9
- import {modelsCache, runCryptoWorkerTask} from "./data.core.js";
10
- import * as tar from "tar";
11
- import {getCollection, getDatabase, getUserCollectionName, modelsCollection} from "../mongodb.js";
12
- import {removeFile} from "../file.js";
13
- import {cancelAlerts, scheduleAlerts} from "./data.scheduling.js";
14
- import {dbName} from "../../constants.js";
15
- import {scheduleWorkflowTriggers} from "../workflow.js";
16
- import crypto from "node:crypto";
17
- import process from "node:process";
18
- import {promisify} from "node:util";
19
- import {execFile} from "node:child_process";
20
- import AWS from "aws-sdk";
21
- import {Logger} from "../../gameObject.js";
22
-
23
- const getBackupDir = () => process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
24
- const execFileAsync = promisify(execFile);
25
-
26
- let engine, logger;
27
- export function onInit(defaultEngine) {
28
- engine = defaultEngine;
29
- logger = engine.getComponent(Logger);
30
- }
31
-
32
- export const jobDumpUserData = async () => {
33
-
34
- try {
35
-
36
- const primalsDb = MongoClient.db("primals");
37
-
38
- let usersCollection = primalsDb.collection("users");
39
- const users = await usersCollection.find().toArray();
40
-
41
- users.forEach((user) => {
42
- if (isDemoUser(user) && Config.Get("useDemoAccounts"))
43
- return;
44
- try {
45
- dumpUserData(user).catch(e => {
46
- Event.Trigger("OnUserDataDumped", "event", "system", engine);
47
- })
48
- } catch (ignored) {
49
-
50
- }
51
- });
52
-
53
- } catch (e) {
54
- console.error(e);
55
- }
56
- }
57
- let restoreRequests = {};
58
- export const validateRestoreRequest = (username, token) => {
59
- const request = restoreRequests[username];
60
- if (!request) {
61
- return {error: 'Invalid username.'};
62
- }
63
-
64
- if (request.token !== token) {
65
- return {error: 'Invalid token.'};
66
- }
67
-
68
- if (request.expiresAt < new Date()) {
69
- delete restoreRequests[username];
70
- return {error: 'Token has expired.'};
71
- }
72
-
73
- delete restoreRequests[username]; // Remove the request after validation
74
- return {success: true};
75
- };
76
- export const loadFromDump = async (user, options = {}) => {
77
- const {modelsOnly = false} = options;
78
- const action = modelsOnly ? 'restore-models' : 'full-restore';
79
- logger.info(`[${action}] Starting for user: ${user.username}`);
80
-
81
- const encryptedKey = readKeyFromFile(user);
82
- if (!encryptedKey) {
83
- throw new Error("No encryption key found for this user. Cannot restore.");
84
- }
85
-
86
- const userId = getObjectHash({user: user.username});
87
- const backupDir = getBackupDir();
88
- let backupFilePath = ''; // Will hold the path to the archive to be restored
89
- let isTempFile = false; // Flag to know if we need to delete the file later
90
-
91
- const tmpRestoreDir = path.join(backupDir, `tmp_restore_${userId}_${Date.now()}`);
92
-
93
- try {
94
- const s3Config = await getUserS3Config(user);
95
- // --- NEW LOGIC: Check for S3 config first ---
96
- if (s3Config && s3Config.bucketName && s3Config.accessKeyId && s3Config.secretAccessKey) {
97
- logger.info(`[${action}] S3 config found for user. Searching for backups in bucket: ${s3Config.bucketName}`);
98
-
99
- const s3Backups = await listS3Backups(s3Config);
100
- const userBackups = s3Backups
101
- .filter(f => f.filename.startsWith(`backup_${userId}_`))
102
- .sort((a, b) => b.timestamp - a.timestamp);
103
-
104
- if (userBackups.length === 0) {
105
- throw new Error(`No S3 backups found for user ${user.username} in bucket ${s3Config.bucketName}.`);
106
- }
107
-
108
- const latestBackup = userBackups[0];
109
- logger.info(`[${action}] Found latest S3 backup: ${latestBackup.key}. Downloading...`);
110
-
111
- // Download the file to a temporary location
112
- backupFilePath = path.join(backupDir, latestBackup.filename);
113
- isTempFile = true;
114
- await downloadFromS3(s3Config, latestBackup.key, backupFilePath);
115
- logger.info(`[${action}] S3 backup downloaded to ${backupFilePath}.`);
116
-
117
- } else {
118
- // --- FALLBACK LOGIC: Look for local backups ---
119
- logger.info(`[${action}] No S3 config. Searching for local backups.`);
120
- const backupFilenameRegex = new RegExp(`^backup_${userId}_(\\d+)\\.tar\\.gz$`);
121
- const backupFiles = fs.readdirSync(backupDir).filter(filename => backupFilenameRegex.test(filename));
122
- if (backupFiles.length === 0) {
123
- throw new Error(`No local backup files found for user ${user.username}.`);
124
- }
125
- const latestBackupFile = backupFiles.sort((a, b) => parseInt(b.match(backupFilenameRegex)[1], 10) - parseInt(a.match(backupFilenameRegex)[1], 10))[0];
126
- backupFilePath = path.join(backupDir, latestBackupFile);
127
- }
128
-
129
- // --- The rest of the logic remains the same, operating on backupFilePath ---
130
-
131
- await runCryptoWorkerTask('decrypt', {filePath: backupFilePath, password: encryptedKey});
132
-
133
- if (!fs.existsSync(tmpRestoreDir)) {
134
- fs.mkdirSync(tmpRestoreDir, {recursive: true});
135
- }
136
- await tar.extract({file: backupFilePath, gzip: true, C: tmpRestoreDir, sync: true});
137
-
138
- // ... (Cleaning logic: deleteMany, removeFile, cancelAlerts) ...
139
- const datasCollection = getCollection(Config.Get('dataCollection', 'datas'));
140
- if (modelsOnly) {
141
- await modelsCollection.deleteMany({_user: user.username});
142
- } else {
143
- await datasCollection.deleteMany({_user: user.username});
144
- await modelsCollection.deleteMany({_user: user.username});
145
-
146
- const filesCollection = getCollection("files");
147
- const userFiles = await filesCollection.find({user: user.username, _model: "privateFile"}).toArray();
148
- for (const file of userFiles) {
149
- await removeFile(file.guid, user).catch(e => logger.error(e.message));
150
- }
151
- await cancelAlerts(user);
152
- logger.info(`[${action}] Cleaned existing data, models, files, and alerts for user ${user.username}.`);
153
- }
154
-
155
- // --- EXÉCUTION DE MONGORESTORE ---
156
- const d = Config.Get('dbName', dbName);
157
- const restoreSourceDir = path.join(tmpRestoreDir, d);
158
- if (!fs.existsSync(restoreSourceDir)) {
159
- throw new Error(`Restore source directory (${restoreSourceDir}) not found.`);
160
- }
161
-
162
- let command;
163
- const args = [
164
- '--uri', dbUrl,
165
- '--db', d
166
- ];
167
-
168
- if (modelsOnly) {
169
- args.push('--nsInclude', `${d}.models`);
170
- } else {
171
- // mongorestore accepte plusieurs fois l'option --nsInclude
172
- args.push('--nsInclude', `${d}.${Config.Get('dataCollection', 'datas')}`);
173
- args.push('--nsInclude', `${d}.models`);
174
- }
175
- // Le répertoire source est le dernier argument
176
- args.push(restoreSourceDir);
177
-
178
-
179
- logger.info(`[${action}] Executing restore command: ${command}`);
180
- await execFileAsync('mongorestore', args);
181
-
182
- // ... (Post-restore tasks) ...
183
- await scheduleAlerts();
184
- await scheduleWorkflowTriggers();
185
- modelsCache.flushAll();
186
-
187
- logger.info(`[${action}] Restore successful for user ${user.username}.`);
188
- await Event.Trigger("OnDataRestored", "event", "system");
189
-
190
- } finally {
191
- // --- GUARANTEED CLEANUP ---
192
- if (fs.existsSync(tmpRestoreDir)) {
193
- await fs.promises.rm(tmpRestoreDir, {recursive: true, force: true});
194
- }
195
-
196
- // Re-encrypt the original file if it's not a temporary one
197
- if (fs.existsSync(backupFilePath) && !isTempFile) {
198
- await runCryptoWorkerTask('encrypt', {filePath: backupFilePath, password: encryptedKey});
199
- }
200
-
201
- // If we downloaded a temp file from S3, delete it
202
- if (fs.existsSync(backupFilePath) && isTempFile) {
203
- fs.unlinkSync(backupFilePath);
204
- logger.info(`[${action}] Deleted temporary downloaded backup file: ${backupFilePath}`);
205
- }
206
- }
207
- };
208
- // Fonction pour générer une clé aléatoire et la stocker dans un fichier
209
- const generateAndStoreKey = (user) => {
210
- const backupDir = getBackupDir();
211
- const keyFile = path.join(backupDir, getObjectHash({id: getUserId(user)}) + '_encryption.key');
212
- const key = crypto.randomBytes(16).toString('hex');
213
- fs.writeFileSync(keyFile, key, {mode: 0o600}); // Permissions strictes
214
- return key;
215
- };
216
- // Fonction pour lire la clé depuis le fichier
217
- const readKeyFromFile = (user) => {
218
- const backupDir = getBackupDir();
219
- const keyFile = path.join(backupDir, getObjectHash({id: getUserId(user)}) + '_encryption.key');
220
- if (fs.existsSync(keyFile)) {
221
- return fs.readFileSync(keyFile, 'utf8');
222
- }
223
- return null;
224
- };
225
- export const dumpUserData = async (user) => {
226
- const s3Config = await getUserS3Config(user);
227
- const backupDir = getBackupDir();
228
- const userId = getObjectHash({user: user.username});
229
- const backupFilename = `backup_${userId}`;
230
- const timestamp = Date.now();
231
-
232
- // Déclarer les chemins ici pour qu'ils soient accessibles dans tout le scope de la fonction
233
- const localTempDumpDir = path.join(backupDir, `${backupFilename}_${timestamp}_temp`);
234
- const finalArchiveName = `${backupFilename}_${timestamp}.tar.gz`;
235
- const localArchivePath = path.join(backupDir, finalArchiveName);
236
-
237
- let encryptedKey = readKeyFromFile(user);
238
- if (!encryptedKey) {
239
- encryptedKey = generateAndStoreKey(user);
240
- }
241
-
242
- try {
243
- const backupFrequency = await engine.userProvider.getBackupFrequency(user);
244
- logger.info(`Fréquence de sauvegarde : ${backupFrequency}.`);
245
-
246
-
247
- const d = Config.Get('dbName', dbName);
248
- const collections = await getDatabase().listCollections().toArray();
249
- for (const collection of collections) {
250
- const collsToBackup = [await getUserCollectionName(user), 'models'];
251
- if (collsToBackup.includes(collection.name)) {
252
- const query = {_user: user.username};
253
- const args = [
254
- '--uri', dbUrl,
255
- '--db', d,
256
- '--out', localTempDumpDir,
257
- '--collection', collection.name,
258
- '--query', JSON.stringify(query)
259
- ];
260
- logger.info(`Exécution de la commande : mongodump ${args.join(' ')}`);
261
- await execFileAsync('mongodump', args);
262
- }
263
- }
264
- const dumpSourceDir = path.join(localTempDumpDir, d);
265
- if (fs.existsSync(dumpSourceDir)) {
266
- await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [d]);
267
- logger.info(`Archive de sauvegarde locale créée : ${localArchivePath}`);
268
- } else {
269
- logger.warn(`Le répertoire de dump ${dumpSourceDir} était vide. Aucune archive n'a créée.`);
270
- return Promise.resolve();
271
- }
272
-
273
- await runCryptoWorkerTask('encrypt', {filePath: localArchivePath, password: encryptedKey});
274
-
275
- try {
276
- // Attempt the S3 upload
277
- await uploadToS3(s3Config, localArchivePath, finalArchiveName);
278
- // ONLY if the upload succeeds, delete the local file.
279
- fs.unlinkSync(localArchivePath);
280
- logger.info(`Local archive ${finalArchiveName} deleted after successful S3 upload.`);
281
- } catch (e) {
282
- }
283
-
284
- logger.info(`Sauvegarde réussie pour l'utilisateur ${user.username}.`);
285
- await manageBackupRotation(user, await engine.userProvider.getBackupFrequency(user), s3Config);
286
-
287
- } catch (error) {
288
- logger.error(`Erreur lors de la sauvegarde pour l'utilisateur ${user.username}:`, error);
289
- // Nettoyage de l'archive si elle a été créée avant l'erreur
290
- if (fs.existsSync(localArchivePath)) {
291
- fs.unlinkSync(localArchivePath);
292
- }
293
- throw error; // Relancer l'erreur pour que l'appelant soit informé
294
- } finally {
295
- // --- NETTOYAGE GARANTI ---
296
- // Ce bloc s'exécute toujours, que la sauvegarde réussisse ou échoue.
297
- if (fs.existsSync(localTempDumpDir)) {
298
- fs.rmSync(localTempDumpDir, {recursive: true, force: true});
299
- logger.info(`Répertoire de dump temporaire ${localTempDumpDir} supprimé.`);
300
- }
301
- }
302
- };
303
-
304
- async function manageBackupRotation(user, backupFrequency, s3Config = null) { // Accepter s3Config
305
- const userId = getObjectHash({user: user.username});
306
- let filesToManage = [];
307
-
308
- if (s3Config && s3Config.bucketName && s3Config.accessKeyId && s3Config.secretAccessKey) {
309
- logger.info(`Gestion de la rotation des sauvegardes S3 pour ${userId}.`);
310
- const s3Backups = await listS3Backups(s3Config);
311
- // Filtrer pour ne garder que les backups de cet utilisateur et trier
312
- filesToManage = s3Backups
313
- .filter(f => f.filename.startsWith(`backup_${userId}_`) && f.filename.endsWith('.tar.gz'))
314
- .map(f => ({
315
- name: f.filename,
316
- key: f.key,
317
- timestamp: f.timestamp
318
- })) // listS3Backups devrait fournir le timestamp
319
- .sort((a, b) => b.timestamp - a.timestamp); // Tri décroissant (plus récent en premier)
320
-
321
- } else {
322
- logger.info(`Gestion de la rotation des sauvegardes locales pour ${userId}.`);
323
- const backupDir = getBackupDir();
324
- const localFiles = fs.readdirSync(backupDir);
325
- filesToManage = localFiles
326
- .filter(f => !fs.lstatSync(path.join(backupDir, f)).isDirectory() && f.startsWith(`backup_${userId}_`) && f.endsWith('.tar.gz'))
327
- .map(f => {
328
- const match = f.match(/_(\d+)\.tar\.gz$/);
329
- return {name: f, key: path.join(backupDir, f), timestamp: match ? parseInt(match[1], 10) : 0};
330
- })
331
- .sort((a, b) => b.timestamp - a.timestamp); // Tri décroissant
332
- }
333
-
334
- let maxFilesToKeep;
335
- // ... (ta logique existante pour maxFilesToKeep basée sur backupFrequency)
336
- switch (backupFrequency) {
337
- case 'daily': // Premium
338
- maxFilesToKeep = 7; // Garder 7 jours
339
- break;
340
- case 'weekly': // Standard
341
- maxFilesToKeep = 4; // Garder 4 semaines
342
- break;
343
- case 'monthly': // Free
344
- default:
345
- maxFilesToKeep = 2; // Garder 2 mois
346
- break;
347
- }
348
- logger.info(`Rotation pour ${userId}: fréquence ${backupFrequency}, garde ${maxFilesToKeep} sauvegardes.`);
349
-
350
-
351
- if (filesToManage.length > maxFilesToKeep) {
352
- const filesToDelete = filesToManage.slice(maxFilesToKeep);
353
- logger.info(`Suppression de ${filesToDelete.length} anciennes sauvegardes pour ${userId}.`);
354
-
355
- const deletionPromises = filesToDelete.map(async (fileInfo) => {
356
- try {
357
- if (s3Config && s3Config.bucketName) {
358
- const s3 = new AWS.S3({ /* ... config ... */
359
- accessKeyId: s3Config.accessKeyId,
360
- secretAccessKey: s3Config.secretAccessKey,
361
- region: s3Config.region
362
- });
363
- await s3.deleteObject({Bucket: s3Config.bucketName, Key: fileInfo.key}).promise();
364
- logger.info(`Ancienne sauvegarde S3 supprimée : ${fileInfo.key}`);
365
- } else {
366
- await fs.promises.unlink(fileInfo.key); // key est le chemin complet pour les fichiers locaux
367
- logger.info(`Ancienne sauvegarde locale supprimée : ${fileInfo.name}`);
368
- }
369
- } catch (err) {
370
- logger.error(`Erreur lors de la suppression de l'ancienne sauvegarde ${fileInfo.name || fileInfo.key}:`, err);
371
- }
372
- });
373
- await Promise.allSettled(deletionPromises);
374
- } else {
375
- logger.info(`Aucune ancienne sauvegarde à supprimer pour ${userId} (total: ${filesToManage.length}, garde: ${maxFilesToKeep}).`);
376
- }
1
+ import {dbUrl, MongoClient, MongoDatabase} from "../../engine.js";
2
+ import {getUserId, isDemoUser} from "../../data.js";
3
+ import {Config} from "../../config.js";
4
+ import {Event} from "../../events.js";
5
+ import {getObjectHash} from "../../core.js";
6
+ import path from "node:path";
7
+ import {downloadFromS3, getUserS3Config, listS3Backups, uploadToS3} from "../bucket.js";
8
+ import fs from "node:fs";
9
+ import {modelsCache, runCryptoWorkerTask} from "./data.core.js";
10
+ import * as tar from "tar";
11
+ import {getCollection, getDatabase, getUserCollectionName, modelsCollection} from "../mongodb.js";
12
+ import {removeFile} from "../file.js";
13
+ import {cancelAlerts, scheduleAlerts} from "./data.scheduling.js";
14
+ import {dbName} from "../../constants.js";
15
+ import {scheduleWorkflowTriggers} from "../workflow.js";
16
+ import crypto from "node:crypto";
17
+ import process from "node:process";
18
+ import {promisify} from "node:util";
19
+ import {execFile} from "node:child_process";
20
+ import AWS from "aws-sdk";
21
+ import {Logger} from "../../gameObject.js";
22
+
23
+ const getBackupDir = () => process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
24
+ const execFileAsync = promisify(execFile);
25
+
26
+ let engine, logger;
27
+ export function onInit(defaultEngine) {
28
+ engine = defaultEngine;
29
+ logger = engine.getComponent(Logger);
30
+ }
31
+
32
+ export const jobDumpUserData = async () => {
33
+
34
+ try {
35
+
36
+ const primalsDb = MongoClient.db("primals");
37
+
38
+ let usersCollection = primalsDb.collection("users");
39
+ const users = await usersCollection.find().toArray();
40
+
41
+ users.forEach((user) => {
42
+ if (isDemoUser(user) && Config.Get("useDemoAccounts"))
43
+ return;
44
+ try {
45
+ dumpUserData(user).catch(e => {
46
+ Event.Trigger("OnUserDataDumped", "event", "system", engine);
47
+ })
48
+ } catch (ignored) {
49
+
50
+ }
51
+ });
52
+
53
+ } catch (e) {
54
+ console.error(e);
55
+ }
56
+ }
57
+ let restoreRequests = {};
58
+ export const validateRestoreRequest = (username, token) => {
59
+ const request = restoreRequests[username];
60
+ if (!request) {
61
+ return {error: 'Invalid username.'};
62
+ }
63
+
64
+ if (request.token !== token) {
65
+ return {error: 'Invalid token.'};
66
+ }
67
+
68
+ if (request.expiresAt < new Date()) {
69
+ delete restoreRequests[username];
70
+ return {error: 'Token has expired.'};
71
+ }
72
+
73
+ delete restoreRequests[username]; // Remove the request after validation
74
+ return {success: true};
75
+ };
76
+ export const loadFromDump = async (user, options = {}) => {
77
+ const {modelsOnly = false} = options;
78
+ const action = modelsOnly ? 'restore-models' : 'full-restore';
79
+ logger.info(`[${action}] Starting for user: ${user.username}`);
80
+
81
+ const encryptedKey = readKeyFromFile(user);
82
+ if (!encryptedKey) {
83
+ throw new Error("No encryption key found for this user. Cannot restore.");
84
+ }
85
+
86
+ const userId = getObjectHash({user: user.username});
87
+ const backupDir = getBackupDir();
88
+ let backupFilePath = ''; // Will hold the path to the archive to be restored
89
+ let isTempFile = false; // Flag to know if we need to delete the file later
90
+
91
+ const tmpRestoreDir = path.join(backupDir, `tmp_restore_${userId}_${Date.now()}`);
92
+
93
+ try {
94
+ const s3Config = await getUserS3Config(user);
95
+ // --- NEW LOGIC: Check for S3 config first ---
96
+ if (s3Config && s3Config.bucketName && s3Config.accessKeyId && s3Config.secretAccessKey) {
97
+ logger.info(`[${action}] S3 config found for user. Searching for backups in bucket: ${s3Config.bucketName}`);
98
+
99
+ const s3Backups = await listS3Backups(s3Config);
100
+ const userBackups = s3Backups
101
+ .filter(f => f.filename.startsWith(`backup_${userId}_`))
102
+ .sort((a, b) => b.timestamp - a.timestamp);
103
+
104
+ if (userBackups.length === 0) {
105
+ throw new Error(`No S3 backups found for user ${user.username} in bucket ${s3Config.bucketName}.`);
106
+ }
107
+
108
+ const latestBackup = userBackups[0];
109
+ logger.info(`[${action}] Found latest S3 backup: ${latestBackup.key}. Downloading...`);
110
+
111
+ // Download the file to a temporary location
112
+ backupFilePath = path.join(backupDir, latestBackup.filename);
113
+ isTempFile = true;
114
+ await downloadFromS3(s3Config, latestBackup.key, backupFilePath);
115
+ logger.info(`[${action}] S3 backup downloaded to ${backupFilePath}.`);
116
+
117
+ } else {
118
+ // --- FALLBACK LOGIC: Look for local backups ---
119
+ logger.info(`[${action}] No S3 config. Searching for local backups.`);
120
+ const backupFilenameRegex = new RegExp(`^backup_${userId}_(\\d+)\\.tar\\.gz$`);
121
+ const backupFiles = fs.readdirSync(backupDir).filter(filename => backupFilenameRegex.test(filename));
122
+ if (backupFiles.length === 0) {
123
+ throw new Error(`No local backup files found for user ${user.username}.`);
124
+ }
125
+ const latestBackupFile = backupFiles.sort((a, b) => parseInt(b.match(backupFilenameRegex)[1], 10) - parseInt(a.match(backupFilenameRegex)[1], 10))[0];
126
+ backupFilePath = path.join(backupDir, latestBackupFile);
127
+ }
128
+
129
+ // --- The rest of the logic remains the same, operating on backupFilePath ---
130
+
131
+ await runCryptoWorkerTask('decrypt', {filePath: backupFilePath, password: encryptedKey});
132
+
133
+ if (!fs.existsSync(tmpRestoreDir)) {
134
+ fs.mkdirSync(tmpRestoreDir, {recursive: true});
135
+ }
136
+ await tar.extract({file: backupFilePath, gzip: true, C: tmpRestoreDir, sync: true});
137
+
138
+ // ... (Cleaning logic: deleteMany, removeFile, cancelAlerts) ...
139
+ const datasCollection = getCollection(Config.Get('dataCollection', 'datas'));
140
+ if (modelsOnly) {
141
+ await modelsCollection.deleteMany({_user: user.username});
142
+ } else {
143
+ await datasCollection.deleteMany({_user: user.username});
144
+ await modelsCollection.deleteMany({_user: user.username});
145
+
146
+ const filesCollection = getCollection("files");
147
+ const userFiles = await filesCollection.find({user: user.username, _model: "privateFile"}).toArray();
148
+ for (const file of userFiles) {
149
+ await removeFile(file.guid, user).catch(e => logger.error(e.message));
150
+ }
151
+ await cancelAlerts(user);
152
+ logger.info(`[${action}] Cleaned existing data, models, files, and alerts for user ${user.username}.`);
153
+ }
154
+
155
+ // --- EXÉCUTION DE MONGORESTORE ---
156
+ const d = Config.Get('dbName', dbName);
157
+ const restoreSourceDir = path.join(tmpRestoreDir, d);
158
+ if (!fs.existsSync(restoreSourceDir)) {
159
+ throw new Error(`Restore source directory (${restoreSourceDir}) not found.`);
160
+ }
161
+
162
+ const args = [
163
+ '--uri', dbUrl,
164
+ '--db', d
165
+ ];
166
+
167
+ // --- AJOUT DE LA LOGIQUE TLS ---
168
+ const isTlsActive = !(!process.env.TLS || ["0", "false"].includes(process.env.TLS.toLowerCase()));
169
+ if (isTlsActive) {
170
+ args.push('--ssl');
171
+ if (process.env.CA_CERT) {
172
+ args.push('--sslCAFile', process.env.CA_CERT);
173
+ }
174
+ if (process.env.CERT_KEY) {
175
+ args.push('--sslPEMKeyPassword', process.env.CERT_KEY);
176
+ }
177
+ }
178
+
179
+ if (modelsOnly) {
180
+ args.push('--nsInclude', `${d}.models`);
181
+ } else {
182
+ // mongorestore accepte plusieurs fois l'option --nsInclude
183
+ args.push('--nsInclude', `${d}.${Config.Get('dataCollection', 'datas')}`);
184
+ args.push('--nsInclude', `${d}.models`);
185
+ }
186
+ // Le répertoire source est le dernier argument
187
+ args.push(restoreSourceDir);
188
+
189
+
190
+ logger.info(`[${action}] Executing mongorestore command...`);
191
+ await execFileAsync('mongorestore', args);
192
+
193
+ // ... (Post-restore tasks) ...
194
+ await scheduleAlerts();
195
+ await scheduleWorkflowTriggers();
196
+ modelsCache.flushAll();
197
+
198
+ logger.info(`[${action}] Restore successful for user ${user.username}.`);
199
+ await Event.Trigger("OnDataRestored", "event", "system");
200
+
201
+ } finally {
202
+ // --- GUARANTEED CLEANUP ---
203
+ if (fs.existsSync(tmpRestoreDir)) {
204
+ await fs.promises.rm(tmpRestoreDir, {recursive: true, force: true});
205
+ }
206
+
207
+ // Re-encrypt the original file if it's not a temporary one
208
+ if (fs.existsSync(backupFilePath) && !isTempFile) {
209
+ await runCryptoWorkerTask('encrypt', {filePath: backupFilePath, password: encryptedKey});
210
+ }
211
+
212
+ // If we downloaded a temp file from S3, delete it
213
+ if (fs.existsSync(backupFilePath) && isTempFile) {
214
+ fs.unlinkSync(backupFilePath);
215
+ logger.info(`[${action}] Deleted temporary downloaded backup file: ${backupFilePath}`);
216
+ }
217
+ }
218
+ };
219
+ // Fonction pour générer une clé aléatoire et la stocker dans un fichier
220
+ const generateAndStoreKey = (user) => {
221
+ const backupDir = getBackupDir();
222
+ const keyFile = path.join(backupDir, getObjectHash({id: getUserId(user)}) + '_encryption.key');
223
+ const key = crypto.randomBytes(16).toString('hex');
224
+ fs.writeFileSync(keyFile, key, {mode: 0o600}); // Permissions strictes
225
+ return key;
226
+ };
227
+ // Fonction pour lire la clé depuis le fichier
228
+ const readKeyFromFile = (user) => {
229
+ const backupDir = getBackupDir();
230
+ const keyFile = path.join(backupDir, getObjectHash({id: getUserId(user)}) + '_encryption.key');
231
+ if (fs.existsSync(keyFile)) {
232
+ return fs.readFileSync(keyFile, 'utf8');
233
+ }
234
+ return null;
235
+ };
236
+ export const dumpUserData = async (user) => {
237
+ const s3Config = await getUserS3Config(user);
238
+ const backupDir = getBackupDir();
239
+ const userId = getObjectHash({user: user.username});
240
+ const backupFilename = `backup_${userId}`;
241
+ const timestamp = Date.now();
242
+
243
+ // Déclarer les chemins ici pour qu'ils soient accessibles dans tout le scope de la fonction
244
+ const localTempDumpDir = path.join(backupDir, `${backupFilename}_${timestamp}_temp`);
245
+ const finalArchiveName = `${backupFilename}_${timestamp}.tar.gz`;
246
+ const localArchivePath = path.join(backupDir, finalArchiveName);
247
+
248
+ let encryptedKey = readKeyFromFile(user);
249
+ if (!encryptedKey) {
250
+ encryptedKey = generateAndStoreKey(user);
251
+ }
252
+
253
+ try {
254
+ const backupFrequency = await engine.userProvider.getBackupFrequency(user);
255
+ logger.info(`Fréquence de sauvegarde : ${backupFrequency}.`);
256
+
257
+
258
+ const d = Config.Get('dbName', dbName);
259
+ const collections = await getDatabase().listCollections().toArray();
260
+ for (const collection of collections) {
261
+ const collsToBackup = [await getUserCollectionName(user), 'models'];
262
+ if (collsToBackup.includes(collection.name)) {
263
+ const query = {_user: user.username};
264
+
265
+ const args = [
266
+ '--uri', dbUrl,
267
+ '--db', d,
268
+ '--out', localTempDumpDir,
269
+ '--collection', collection.name,
270
+ '--query', JSON.stringify(query)
271
+ ];
272
+
273
+ // --- AJOUT DE LA LOGIQUE TLS ---
274
+ const isTlsActive = !(!process.env.TLS || ["0", "false"].includes(process.env.TLS.toLowerCase()));
275
+ if (isTlsActive) {
276
+ args.push('--ssl');
277
+ if (process.env.CA_CERT) {
278
+ args.push('--sslCAFile', process.env.CA_CERT);
279
+ }
280
+ if (process.env.CERT_KEY) {
281
+ args.push('--sslPEMKeyPassword', process.env.CERT_KEY);
282
+ }
283
+ }
284
+ logger.info(`Exécution de la commande : mongodump ${args.join(' ')}`);
285
+ await execFileAsync('mongodump', args);
286
+ }
287
+ }
288
+ const dumpSourceDir = path.join(localTempDumpDir, d);
289
+ if (fs.existsSync(dumpSourceDir)) {
290
+ await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [d]);
291
+ logger.info(`Archive de sauvegarde locale créée : ${localArchivePath}`);
292
+ } else {
293
+ logger.warn(`Le répertoire de dump ${dumpSourceDir} était vide. Aucune archive n'a créée.`);
294
+ return Promise.resolve();
295
+ }
296
+
297
+ await runCryptoWorkerTask('encrypt', {filePath: localArchivePath, password: encryptedKey});
298
+
299
+ try {
300
+ // Attempt the S3 upload
301
+ await uploadToS3(s3Config, localArchivePath, finalArchiveName);
302
+ // ONLY if the upload succeeds, delete the local file.
303
+ fs.unlinkSync(localArchivePath);
304
+ logger.info(`Local archive ${finalArchiveName} deleted after successful S3 upload.`);
305
+ } catch (e) {
306
+ }
307
+
308
+ logger.info(`Sauvegarde réussie pour l'utilisateur ${user.username}.`);
309
+ await manageBackupRotation(user, await engine.userProvider.getBackupFrequency(user), s3Config);
310
+
311
+ } catch (error) {
312
+ logger.error(`Erreur lors de la sauvegarde pour l'utilisateur ${user.username}:`, error);
313
+ // Nettoyage de l'archive si elle a été créée avant l'erreur
314
+ if (fs.existsSync(localArchivePath)) {
315
+ fs.unlinkSync(localArchivePath);
316
+ }
317
+ throw error; // Relancer l'erreur pour que l'appelant soit informé
318
+ } finally {
319
+ // --- NETTOYAGE GARANTI ---
320
+ // Ce bloc s'exécute toujours, que la sauvegarde réussisse ou échoue.
321
+ if (fs.existsSync(localTempDumpDir)) {
322
+ fs.rmSync(localTempDumpDir, {recursive: true, force: true});
323
+ logger.info(`Répertoire de dump temporaire ${localTempDumpDir} supprimé.`);
324
+ }
325
+ }
326
+ };
327
+
328
+ async function manageBackupRotation(user, backupFrequency, s3Config = null) { // Accepter s3Config
329
+ const userId = getObjectHash({user: user.username});
330
+ let filesToManage = [];
331
+
332
+ if (s3Config && s3Config.bucketName && s3Config.accessKeyId && s3Config.secretAccessKey) {
333
+ logger.info(`Gestion de la rotation des sauvegardes S3 pour ${userId}.`);
334
+ const s3Backups = await listS3Backups(s3Config);
335
+ // Filtrer pour ne garder que les backups de cet utilisateur et trier
336
+ filesToManage = s3Backups
337
+ .filter(f => f.filename.startsWith(`backup_${userId}_`) && f.filename.endsWith('.tar.gz'))
338
+ .map(f => ({
339
+ name: f.filename,
340
+ key: f.key,
341
+ timestamp: f.timestamp
342
+ })) // listS3Backups devrait fournir le timestamp
343
+ .sort((a, b) => b.timestamp - a.timestamp); // Tri décroissant (plus récent en premier)
344
+
345
+ } else {
346
+ logger.info(`Gestion de la rotation des sauvegardes locales pour ${userId}.`);
347
+ const backupDir = getBackupDir();
348
+ const localFiles = fs.readdirSync(backupDir);
349
+ filesToManage = localFiles
350
+ .filter(f => !fs.lstatSync(path.join(backupDir, f)).isDirectory() && f.startsWith(`backup_${userId}_`) && f.endsWith('.tar.gz'))
351
+ .map(f => {
352
+ const match = f.match(/_(\d+)\.tar\.gz$/);
353
+ return {name: f, key: path.join(backupDir, f), timestamp: match ? parseInt(match[1], 10) : 0};
354
+ })
355
+ .sort((a, b) => b.timestamp - a.timestamp); // Tri décroissant
356
+ }
357
+
358
+ let maxFilesToKeep;
359
+ // ... (ta logique existante pour maxFilesToKeep basée sur backupFrequency)
360
+ switch (backupFrequency) {
361
+ case 'daily': // Premium
362
+ maxFilesToKeep = 7; // Garder 7 jours
363
+ break;
364
+ case 'weekly': // Standard
365
+ maxFilesToKeep = 4; // Garder 4 semaines
366
+ break;
367
+ case 'monthly': // Free
368
+ default:
369
+ maxFilesToKeep = 2; // Garder 2 mois
370
+ break;
371
+ }
372
+ logger.info(`Rotation pour ${userId}: fréquence ${backupFrequency}, garde ${maxFilesToKeep} sauvegardes.`);
373
+
374
+
375
+ if (filesToManage.length > maxFilesToKeep) {
376
+ const filesToDelete = filesToManage.slice(maxFilesToKeep);
377
+ logger.info(`Suppression de ${filesToDelete.length} anciennes sauvegardes pour ${userId}.`);
378
+
379
+ const deletionPromises = filesToDelete.map(async (fileInfo) => {
380
+ try {
381
+ if (s3Config && s3Config.bucketName) {
382
+ const s3 = new AWS.S3({ /* ... config ... */
383
+ accessKeyId: s3Config.accessKeyId,
384
+ secretAccessKey: s3Config.secretAccessKey,
385
+ region: s3Config.region
386
+ });
387
+ await s3.deleteObject({Bucket: s3Config.bucketName, Key: fileInfo.key}).promise();
388
+ logger.info(`Ancienne sauvegarde S3 supprimée : ${fileInfo.key}`);
389
+ } else {
390
+ await fs.promises.unlink(fileInfo.key); // key est le chemin complet pour les fichiers locaux
391
+ logger.info(`Ancienne sauvegarde locale supprimée : ${fileInfo.name}`);
392
+ }
393
+ } catch (err) {
394
+ logger.error(`Erreur lors de la suppression de l'ancienne sauvegarde ${fileInfo.name || fileInfo.key}:`, err);
395
+ }
396
+ });
397
+ await Promise.allSettled(deletionPromises);
398
+ } else {
399
+ logger.info(`Aucune ancienne sauvegarde à supprimer pour ${userId} (total: ${filesToManage.length}, garde: ${maxFilesToKeep}).`);
400
+ }
377
401
  }