data-primals-engine 1.7.1 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/client/package-lock.json +8430 -8121
- package/client/package.json +7 -4
- package/client/src/APIInfo.jsx +1 -1
- package/client/src/App.jsx +2 -1
- package/client/src/App.scss +1635 -1626
- package/client/src/AssistantChat.jsx +2 -3
- package/client/src/CalendarView.jsx +1 -0
- package/client/src/ContentView.jsx +3 -3
- package/client/src/Dashboard.jsx +5 -2
- package/client/src/DashboardChart.jsx +1 -0
- package/client/src/DashboardFlexViewItem.jsx +1 -0
- package/client/src/DashboardHtmlViewItem.jsx +1 -0
- package/client/src/DashboardView.jsx +2 -0
- package/client/src/DataEditor.jsx +0 -1
- package/client/src/DataImporter.jsx +489 -468
- package/client/src/DataLayout.jsx +25 -23
- package/client/src/DataTable.jsx +6 -5
- package/client/src/Dialog.jsx +92 -90
- package/client/src/Dialog.scss +122 -116
- package/client/src/DisplayFlexNodeRenderer.jsx +1 -0
- package/client/src/DocumentationPageLayout.scss +1 -1
- package/client/src/FlexBuilderControls.jsx +1 -0
- package/client/src/FlexBuilderPreview.jsx +1 -1
- package/client/src/FlexNode.jsx +1 -1
- package/client/src/HistoryDialog.jsx +3 -2
- package/client/src/KPIWidget.jsx +1 -1
- package/client/src/KanbanView.jsx +1 -0
- package/client/src/ModelCreator.jsx +4 -0
- package/client/src/ModelList.jsx +1 -0
- package/client/src/PackGallery.jsx +5 -4
- package/client/src/RTETrans.jsx +1 -1
- package/client/src/RelationField.jsx +2 -0
- package/client/src/RelationSelectorWidget.jsx +2 -0
- package/client/src/RelationValue.jsx +1 -0
- package/client/src/RestoreDialog.jsx +1 -0
- package/client/src/ViewSwitcher.jsx +1 -1
- package/client/src/WorkflowEditor.jsx +3 -0
- package/client/src/contexts/CommandContext.jsx +2 -1
- package/client/src/contexts/ModelContext.jsx +3 -3
- package/client/src/hooks/data.js +1 -0
- package/client/src/hooks/useValidation.js +1 -0
- package/client/src/translations.js +24 -24
- package/client/vite.config.js +31 -30
- package/doc/AI-assistance.md +87 -63
- package/doc/Concepts.md +122 -0
- package/doc/Custom-Endpoints.md +31 -0
- package/doc/Event-system.md +13 -14
- package/doc/Home.md +33 -0
- package/doc/Modules.md +83 -0
- package/doc/Packs-gallery.md +8 -23
- package/doc/Workflows.md +32 -0
- package/doc/automation-workflows.md +141 -102
- package/doc/dashboards-kpis-charts.md +47 -49
- package/doc/data-management.md +126 -120
- package/doc/data-models.md +68 -75
- package/doc/roles-permissions.md +144 -43
- package/doc/sharding-replication.md +158 -0
- package/doc/users.md +54 -30
- package/package.json +27 -17
- package/server.js +37 -37
- package/src/ai.jobs.js +135 -0
- package/src/constants.js +560 -545
- package/src/data.js +521 -518
- package/src/email.js +157 -154
- package/src/engine.js +167 -49
- package/src/gameObject.js +6 -0
- package/src/i18n.js +0 -1
- package/src/modules/assistant/assistant.js +782 -763
- package/src/modules/assistant/constants.js +23 -16
- package/src/modules/assistant/providers.js +77 -37
- package/src/modules/auth-google/index.js +53 -50
- package/src/modules/bucket.js +346 -335
- package/src/modules/data/data.backup.js +400 -376
- package/src/modules/data/data.cluster.js +559 -0
- package/src/modules/data/data.core.js +11 -8
- package/src/modules/data/data.js +311 -311
- package/src/modules/data/data.operations.js +3666 -3555
- package/src/modules/data/data.relations.js +1 -0
- package/src/modules/data/data.replication.js +125 -0
- package/src/modules/data/data.routes.js +2206 -1879
- package/src/modules/data/data.scheduling.js +2 -1
- package/src/modules/file.js +248 -247
- package/src/modules/mongodb.js +76 -73
- package/src/modules/user.js +18 -2
- package/src/modules/worker-script-runner.js +97 -0
- package/src/modules/workflow.js +177 -52
- package/src/packs.js +5701 -5701
- package/src/providers.js +298 -297
- package/src/sso.js +91 -25
- package/test/assistant.test.js +207 -206
- package/test/cluster.test.js +221 -0
- package/test/core.test.js +0 -2
- package/test/data.integration.test.js +1425 -1416
- package/test/import_export.integration.test.js +210 -210
- package/test/replication.test.js +163 -0
- package/test/workflow.actions.integration.test.js +487 -475
- package/test/workflow.integration.test.js +332 -329
- package/doc/core-concepts.md +0 -33
package/src/modules/data/data.js
CHANGED
|
@@ -1,311 +1,311 @@
|
|
|
1
|
-
import {Logger} from "../../gameObject.js";
|
|
2
|
-
import {mkdir} from 'node:fs/promises';
|
|
3
|
-
import {onInit as historyInit} from "./data.history.js";
|
|
4
|
-
import {isDemoUser, isLocalUser} from "../../data.js";
|
|
5
|
-
import {install, storageSafetyMargin} from "../../constants.js";
|
|
6
|
-
import {createCollection, getCollection} from "../mongodb.js";
|
|
7
|
-
import path from "node:path";
|
|
8
|
-
import {isGUID, sequential} from "../../core.js";
|
|
9
|
-
import {Event} from "../../events.js";
|
|
10
|
-
import fs from "node:fs";
|
|
11
|
-
import schedule from "node-schedule";
|
|
12
|
-
import {middleware} from "../../middlewares/middleware-mongodb.js";
|
|
13
|
-
import checkDiskSpace from "check-disk-space";
|
|
14
|
-
import {removeFile} from "../file.js";
|
|
15
|
-
import {hasPermission} from "../user.js";
|
|
16
|
-
import { onInit as userInit } from '../user.js';
|
|
17
|
-
import {registerRoutes} from "./data.routes.js";
|
|
18
|
-
import {mongoDBWhitelist} from "./data.core.js";
|
|
19
|
-
import {onInit as relationsInit} from "./data.relations.js";
|
|
20
|
-
import { onInit as validationInit} from "./data.validation.js";
|
|
21
|
-
import {cancelAlerts, scheduleAlerts, onInit as scheduleInit} from "./data.scheduling.js";
|
|
22
|
-
import {deleteData, installPack, onInit as operationsInit} from "./data.operations.js";
|
|
23
|
-
import {jobDumpUserData, onInit as backupInit} from "./data.backup.js";
|
|
24
|
-
import {Config} from "../../config.js";
|
|
25
|
-
import {profiles} from "../../profiles.js";
|
|
26
|
-
|
|
27
|
-
let engine;
|
|
28
|
-
let logger;
|
|
29
|
-
|
|
30
|
-
const DATA_STORAGE_PATH = path.resolve('./');
|
|
31
|
-
|
|
32
|
-
export const getAPILang = (langs) => {
|
|
33
|
-
if( typeof(langs) !== 'string')
|
|
34
|
-
return 'en';
|
|
35
|
-
const array = (langs || 'en')?.split(/,|;q=/g)
|
|
36
|
-
|
|
37
|
-
let quality
|
|
38
|
-
return array.reverse().reduce((e, val) => {
|
|
39
|
-
if (!isNaN(val)) {
|
|
40
|
-
quality = Number(val);
|
|
41
|
-
} else {
|
|
42
|
-
const [lang, dialect] = val.split('-');
|
|
43
|
-
|
|
44
|
-
e.push({ lang, dialect, quality });
|
|
45
|
-
}
|
|
46
|
-
return e;
|
|
47
|
-
}, []).sort(((p,r) => p.quality < r.quality ? 1 : -1))?.[0].lang.split(/[-_]/)?.[0];
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export async function onInit(defaultEngine) {
|
|
51
|
-
engine = defaultEngine;
|
|
52
|
-
logger = engine.getComponent(Logger);
|
|
53
|
-
|
|
54
|
-
engine.use(middleware({ whitelist: mongoDBWhitelist }));
|
|
55
|
-
|
|
56
|
-
let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection, historyCollection;
|
|
57
|
-
|
|
58
|
-
const i = Config.Get('install', install);
|
|
59
|
-
if( i ) {
|
|
60
|
-
datasCollection = await createCollection(Config.Get('dataCollection', 'datas'));
|
|
61
|
-
historyCollection = await createCollection("history");
|
|
62
|
-
filesCollection = await createCollection("files");
|
|
63
|
-
packsCollection = await createCollection("packs");
|
|
64
|
-
//data
|
|
65
|
-
const indexes = await datasCollection.indexes();
|
|
66
|
-
if (!indexes.find(i => i.name === 'genericPartialIndex')) {
|
|
67
|
-
await datasCollection.createIndex({"$**": 1}, {
|
|
68
|
-
name: 'genericPartialIndex',
|
|
69
|
-
partialFilterExpression: {
|
|
70
|
-
_model: 1,
|
|
71
|
-
_user: 1
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
if (! await datasCollection.indexExists("_hash") ) {
|
|
77
|
-
await datasCollection.createIndex({_hash: 1});
|
|
78
|
-
}
|
|
79
|
-
if (! await datasCollection.indexExists("_model") ) {
|
|
80
|
-
await datasCollection.createIndex({_model: 1});
|
|
81
|
-
}
|
|
82
|
-
if (! await datasCollection.indexExists("_user") ) {
|
|
83
|
-
await datasCollection.createIndex({_user: 1});
|
|
84
|
-
}
|
|
85
|
-
if (!indexes.find(i => i.name === 'modelUserIndex')) {
|
|
86
|
-
await datasCollection.createIndex({_model: 1, _user: 1}, { name: 'modelUserIndex'});
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const jobsCollection = await createCollection("job_locks");
|
|
90
|
-
if (! await jobsCollection.indexExists("jobTTLIndex") ) {
|
|
91
|
-
await jobsCollection.createIndex({ "lockedUntil": 1 }, { name: "jobTTLIndex", expireAfterSeconds: 0 });
|
|
92
|
-
}
|
|
93
|
-
if (! await jobsCollection.indexExists("jobIdUnique") ) {
|
|
94
|
-
await jobsCollection.createIndex({ "jobId": 1 }, { name: "jobIdUnique", unique: true });
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
logger.info("Setting up indexes for 'files' collection...");
|
|
98
|
-
const filesIndexes = await filesCollection.indexes();
|
|
99
|
-
|
|
100
|
-
// Index composé pour les lookups fréquents par GUID et utilisateur
|
|
101
|
-
const compoundGuidUserIndexName = 'file_guid_user_idx';
|
|
102
|
-
if (!filesIndexes.find(i => i.name === compoundGuidUserIndexName)) {
|
|
103
|
-
await filesCollection.createIndex({ guid: 1, user: 1 }, { name: compoundGuidUserIndexName });
|
|
104
|
-
logger.info(`Created compound index '${compoundGuidUserIndexName}' on 'files' collection (guid: 1, user: 1).`);
|
|
105
|
-
} else {
|
|
106
|
-
logger.info(`Index '${compoundGuidUserIndexName}' already exists on 'files' collection.`);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const uniqueGuidIndexName = 'file_guid_unique_idx';
|
|
110
|
-
const existingGuidIndex = filesIndexes.find(i => i.name === uniqueGuidIndexName);
|
|
111
|
-
if (!existingGuidIndex) {
|
|
112
|
-
await filesCollection.createIndex({ guid: 1 }, { name: uniqueGuidIndexName, unique: true });
|
|
113
|
-
logger.info(`Created unique index '${uniqueGuidIndexName}' on 'guid' for 'files' collection.`);
|
|
114
|
-
} else if (existingGuidIndex.name !== uniqueGuidIndexName || !existingGuidIndex.unique) {
|
|
115
|
-
logger.warn(`An index on 'guid' exists for 'files' collection (name: ${existingGuidIndex.name}, unique: ${existingGuidIndex.unique}), but not matching desired spec (name: ${uniqueGuidIndexName}, unique: true). Manual review might be needed.`);
|
|
116
|
-
} else {
|
|
117
|
-
logger.info(`Unique index on 'guid' (name: '${existingGuidIndex.name}') already exists for 'files' collection.`);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
if (! await packsCollection.indexExists("_user") ) {
|
|
121
|
-
await packsCollection.createIndex({_user: 1});
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
// Create the uploads directories
|
|
126
|
-
await mkdir(path.join("uploads", "tmp"),{ recursive: true});
|
|
127
|
-
|
|
128
|
-
}else {
|
|
129
|
-
modelsCollection = getCollection("models");
|
|
130
|
-
datasCollection = getCollection(Config.Get('dataCollection','datas'));
|
|
131
|
-
filesCollection = getCollection("files");
|
|
132
|
-
packsCollection = getCollection("packs");
|
|
133
|
-
historyCollection = getCollection("history");
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Sub modules
|
|
137
|
-
backupInit(defaultEngine);
|
|
138
|
-
historyInit(defaultEngine);
|
|
139
|
-
validationInit(defaultEngine);
|
|
140
|
-
relationsInit(defaultEngine);
|
|
141
|
-
scheduleInit(defaultEngine);
|
|
142
|
-
userInit(defaultEngine);
|
|
143
|
-
operationsInit(defaultEngine);
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
await registerRoutes(engine);
|
|
147
|
-
logger = engine.getComponent(Logger);
|
|
148
|
-
|
|
149
|
-
// set backup scheduler
|
|
150
|
-
schedule.scheduleJob("0 2 * * *", jobDumpUserData);
|
|
151
|
-
//await jobDumpUserData();
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
schedule.scheduleJob("0 0 * * *", async () => {
|
|
155
|
-
const dt = new Date();
|
|
156
|
-
dt.setTime(dt.getTime()-1000*3600*24*14);
|
|
157
|
-
await deleteData("request", {"$lt": ["$timestamp",dt.toISOString()]}, null, false);
|
|
158
|
-
});
|
|
159
|
-
await scheduleAlerts();
|
|
160
|
-
|
|
161
|
-
// Triggers
|
|
162
|
-
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
/**
|
|
167
|
-
* Vérifie si l'ajout de nouvelles données dépasserait la capacité de stockage globale du serveur.
|
|
168
|
-
* @param {number} incomingDataSize - La taille des données entrantes en octets.
|
|
169
|
-
* @returns {Promise<{isSufficient: boolean, free?: number, total?: number, error?: string}>}
|
|
170
|
-
*/
|
|
171
|
-
export async function checkServerCapacity(incomingDataSize = 0) {
|
|
172
|
-
try {
|
|
173
|
-
const diskSpace = await checkDiskSpace(DATA_STORAGE_PATH);
|
|
174
|
-
const { free, size } = diskSpace;
|
|
175
|
-
|
|
176
|
-
const storageMargin = Config.Get('storageSafetyMargin', storageSafetyMargin);
|
|
177
|
-
// Limite maximale d'utilisation du disque (ex: 90% de la taille totale)
|
|
178
|
-
const maxAllowedUsage = size * storageMargin;
|
|
179
|
-
const currentUsage = size - free;
|
|
180
|
-
const projectedUsage = currentUsage + incomingDataSize;
|
|
181
|
-
|
|
182
|
-
if (projectedUsage > maxAllowedUsage) {
|
|
183
|
-
logger.warn(`[checkServerCapacity] Alert: Projected usage (${projectedUsage} bytes) would exceed the server's safety limit (${maxAllowedUsage} bytes).`);
|
|
184
|
-
return {
|
|
185
|
-
isSufficient: false,
|
|
186
|
-
free,
|
|
187
|
-
total: size
|
|
188
|
-
};
|
|
189
|
-
}
|
|
190
|
-
return { isSufficient: true, free, total: size };
|
|
191
|
-
} catch (err) {
|
|
192
|
-
logger.error(`[checkServerCapacity] CRITICAL: Failed to check disk space: ${err.message}. Allowing write operation as a failsafe. Please investigate disk permissions or configuration.`);
|
|
193
|
-
// Failsafe: On autorise l'ure si la vérification échoue, mais on logue une erreur critique.
|
|
194
|
-
return { isSufficient: true, error: 'Could not verify disk space.' };
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
export const getResource = async (guid, user) => {
|
|
200
|
-
if (!guid) throw new Error("Le GUID du fichier est requis.");
|
|
201
|
-
if (!isGUID(guid)) throw new Error("Le GUID du fichier n'est pas valide.");
|
|
202
|
-
|
|
203
|
-
const collection = getCollection("files");
|
|
204
|
-
const file = await collection.findOne({ guid });
|
|
205
|
-
|
|
206
|
-
if (!file) {
|
|
207
|
-
throw new Error("Fichier non trouvé.");
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
// La vérification des permissions reste la même...
|
|
211
|
-
if (user.username !== 'demo' && isLocalUser(user) && !await hasPermission(["API_ADMIN", "API_READ_FILE", `API_READ_FILE_privateFile_${guid}`], user)) {
|
|
212
|
-
if (file.user !== (user._user || user.username)) {
|
|
213
|
-
throw new Error("Vous n'êtes pas autorisé à accéder à ce fichier.");
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
// On retourne des informations différentes selon le type de stockage
|
|
218
|
-
if (file.storage === 's3') {
|
|
219
|
-
return {
|
|
220
|
-
success: true,
|
|
221
|
-
storage: 's3',
|
|
222
|
-
s3Key: file.filename, // 'filename' contient la clé S3
|
|
223
|
-
mimeType: file.mimeType
|
|
224
|
-
// Idlement, on aurait aussi le nom de fichier original ici
|
|
225
|
-
};
|
|
226
|
-
} else { // Par défaut, on considère le stockage local
|
|
227
|
-
// --- CORRECTIF DE SÉCURITÉ ---
|
|
228
|
-
// Ne jamais faire confiance au chemin stocké en base de données.
|
|
229
|
-
// Toujours reconstruire le chemin à partir d'éléments fiables.
|
|
230
|
-
const uploadDir = path.join(process.cwd(), "uploads", "private");
|
|
231
|
-
const safeFilepath = path.join(uploadDir, file.filename); // file.filename est le nom sécurisé (guid.ext)
|
|
232
|
-
|
|
233
|
-
// Vérification supplémentaire pour s'assurer que le chemin résolu est bien dans le répertoire attendu.
|
|
234
|
-
if (!safeFilepath.startsWith(uploadDir)) {
|
|
235
|
-
throw new Error("Tentative d'accès à un chemin non autorisé.");
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
if (!fs.existsSync(safeFilepath)) {
|
|
239
|
-
throw new Error("Fichier non trouvé sur le serveur.");
|
|
240
|
-
}
|
|
241
|
-
return {
|
|
242
|
-
success: true,
|
|
243
|
-
storage: 'local',
|
|
244
|
-
filepath: safeFilepath, // On retourne le chemin sécurisé et reconstruit
|
|
245
|
-
filename: file.filename,
|
|
246
|
-
mimeType: file.mimeType
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
|
-
};
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
export async function handleDemoInitialization(req, res) {
|
|
253
|
-
const user = req.me;
|
|
254
|
-
const body = req.fields;
|
|
255
|
-
const packs = body.packs;
|
|
256
|
-
const models = (Object.keys(profiles).includes(body.profile) && profiles[body.profile].models) || '';
|
|
257
|
-
if (!isDemoUser(user)) {
|
|
258
|
-
return res.status(403).json({ success: false, error: "This action is only for demo users." });
|
|
259
|
-
}
|
|
260
|
-
if (!Array.isArray(models) || models.length === 0) {
|
|
261
|
-
return res.status(400).json({ success: false, error: "A valid 'models' array is required." });
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
logger.info(`[Demo Init] Starting initialization for user '${user.username}' with ${models.length} models.`);
|
|
265
|
-
|
|
266
|
-
try {
|
|
267
|
-
// 1. Nettoyage de l'environnement (inchangé)
|
|
268
|
-
const datasCollection = getCollection(Config.Get('dataCollection',"datas"));
|
|
269
|
-
const modelsCollection = getCollection("models");
|
|
270
|
-
const filesCollection = getCollection("files");
|
|
271
|
-
|
|
272
|
-
await datasCollection.deleteMany({ _user: user.username });
|
|
273
|
-
await modelsCollection.deleteMany({ _user: user.username });
|
|
274
|
-
const files = await filesCollection.find({ user: user.username }).toArray();
|
|
275
|
-
for (const file of files) {
|
|
276
|
-
await removeFile(file.guid, user).catch(e => logger.error(e.message));
|
|
277
|
-
}
|
|
278
|
-
await cancelAlerts(user);
|
|
279
|
-
logger.info(`[Demo Init] Environment cleaned for user '${user.username}'.`);
|
|
280
|
-
|
|
281
|
-
const packToInstall = {
|
|
282
|
-
name: `dynamic-pack-for-${user.username}-${Date.now()}`,
|
|
283
|
-
description: `Dynamically generated pack for profile models.`,
|
|
284
|
-
models: models,
|
|
285
|
-
data: {}
|
|
286
|
-
};
|
|
287
|
-
|
|
288
|
-
logger.info(`[Demo Init] Installing dynamically generated pack with models: [${models.join(', ')}].`);
|
|
289
|
-
|
|
290
|
-
// Create and install pack
|
|
291
|
-
const result = await installPack(packToInstall, user, req.query.lang || 'en');
|
|
292
|
-
|
|
293
|
-
await sequential(packs.map(p => {
|
|
294
|
-
return () => installPack(p, user, req.query.lang || 'en');
|
|
295
|
-
}));
|
|
296
|
-
|
|
297
|
-
if (result.success || result.modifiedCount > 0) {
|
|
298
|
-
|
|
299
|
-
await Event.Trigger('OnDemoUserAdded', "event", "system", req.me.username);
|
|
300
|
-
logger.info(`[Demo Init] Pack installed successfully for user '${user.username}'.`);
|
|
301
|
-
res.status(200).json({ success: true, message: "Demo environment initialized successfully.", summary: result.summary });
|
|
302
|
-
} else {
|
|
303
|
-
logger.error(`[Demo Init] Pack installation failed for user '${user.username}'.`);
|
|
304
|
-
res.status(200).json({ success: false, error: 'Demo pack installation failed.', errors: result.errors });
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
} catch (error) {
|
|
308
|
-
logger.error(`[Demo Init] Critical error during initialization for user '${user.username}':`, error);
|
|
309
|
-
res.status(500).json({ success: false, error: 'An internal server error occurred during initialization.' });
|
|
310
|
-
}
|
|
311
|
-
}
|
|
1
|
+
import {Logger} from "../../gameObject.js";
|
|
2
|
+
import {mkdir} from 'node:fs/promises';
|
|
3
|
+
import {onInit as historyInit} from "./data.history.js";
|
|
4
|
+
import {isDemoUser, isLocalUser} from "../../data.js";
|
|
5
|
+
import {install, storageSafetyMargin} from "../../constants.js";
|
|
6
|
+
import {createCollection, getCollection} from "../mongodb.js";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import {isGUID, sequential} from "../../core.js";
|
|
9
|
+
import {Event} from "../../events.js";
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import schedule from "node-schedule";
|
|
12
|
+
import {middleware} from "../../middlewares/middleware-mongodb.js";
|
|
13
|
+
import checkDiskSpace from "check-disk-space";
|
|
14
|
+
import {removeFile} from "../file.js";
|
|
15
|
+
import {hasPermission} from "../user.js";
|
|
16
|
+
import { onInit as userInit } from '../user.js';
|
|
17
|
+
import {registerRoutes} from "./data.routes.js";
|
|
18
|
+
import {mongoDBWhitelist} from "./data.core.js";
|
|
19
|
+
import {onInit as relationsInit} from "./data.relations.js";
|
|
20
|
+
import { onInit as validationInit} from "./data.validation.js";
|
|
21
|
+
import {cancelAlerts, scheduleAlerts, onInit as scheduleInit} from "./data.scheduling.js";
|
|
22
|
+
import {deleteData, installPack, onInit as operationsInit} from "./data.operations.js";
|
|
23
|
+
import {jobDumpUserData, onInit as backupInit} from "./data.backup.js";
|
|
24
|
+
import {Config} from "../../config.js";
|
|
25
|
+
import {profiles} from "../../profiles.js";
|
|
26
|
+
|
|
27
|
+
let engine;
|
|
28
|
+
let logger;
|
|
29
|
+
|
|
30
|
+
const DATA_STORAGE_PATH = path.resolve('./');
|
|
31
|
+
|
|
32
|
+
export const getAPILang = (langs) => {
|
|
33
|
+
if( typeof(langs) !== 'string')
|
|
34
|
+
return 'en';
|
|
35
|
+
const array = (langs || 'en')?.split(/,|;q=/g)
|
|
36
|
+
|
|
37
|
+
let quality
|
|
38
|
+
return array.reverse().reduce((e, val) => {
|
|
39
|
+
if (!isNaN(val)) {
|
|
40
|
+
quality = Number(val);
|
|
41
|
+
} else {
|
|
42
|
+
const [lang, dialect] = val.split('-');
|
|
43
|
+
|
|
44
|
+
e.push({ lang, dialect, quality });
|
|
45
|
+
}
|
|
46
|
+
return e;
|
|
47
|
+
}, []).sort(((p,r) => p.quality < r.quality ? 1 : -1))?.[0].lang.split(/[-_]/)?.[0];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function onInit(defaultEngine) {
|
|
51
|
+
engine = defaultEngine;
|
|
52
|
+
logger = engine.getComponent(Logger);
|
|
53
|
+
|
|
54
|
+
engine.use(middleware({ whitelist: mongoDBWhitelist }));
|
|
55
|
+
|
|
56
|
+
let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection, historyCollection;
|
|
57
|
+
|
|
58
|
+
const i = Config.Get('install', install);
|
|
59
|
+
if( i ) {
|
|
60
|
+
datasCollection = await createCollection(Config.Get('dataCollection', 'datas'));
|
|
61
|
+
historyCollection = await createCollection("history");
|
|
62
|
+
filesCollection = await createCollection("files");
|
|
63
|
+
packsCollection = await createCollection("packs");
|
|
64
|
+
//data
|
|
65
|
+
const indexes = await datasCollection.indexes();
|
|
66
|
+
if (!indexes.find(i => i.name === 'genericPartialIndex')) {
|
|
67
|
+
await datasCollection.createIndex({"$**": 1}, {
|
|
68
|
+
name: 'genericPartialIndex',
|
|
69
|
+
partialFilterExpression: {
|
|
70
|
+
_model: 1,
|
|
71
|
+
_user: 1
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (! await datasCollection.indexExists("_hash") ) {
|
|
77
|
+
await datasCollection.createIndex({_hash: 1});
|
|
78
|
+
}
|
|
79
|
+
if (! await datasCollection.indexExists("_model") ) {
|
|
80
|
+
await datasCollection.createIndex({_model: 1});
|
|
81
|
+
}
|
|
82
|
+
if (! await datasCollection.indexExists("_user") ) {
|
|
83
|
+
await datasCollection.createIndex({_user: 1});
|
|
84
|
+
}
|
|
85
|
+
if (!indexes.find(i => i.name === 'modelUserIndex')) {
|
|
86
|
+
await datasCollection.createIndex({_model: 1, _user: 1}, { name: 'modelUserIndex'});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const jobsCollection = await createCollection("job_locks");
|
|
90
|
+
if (! await jobsCollection.indexExists("jobTTLIndex") ) {
|
|
91
|
+
await jobsCollection.createIndex({ "lockedUntil": 1 }, { name: "jobTTLIndex", expireAfterSeconds: 0 });
|
|
92
|
+
}
|
|
93
|
+
if (! await jobsCollection.indexExists("jobIdUnique") ) {
|
|
94
|
+
await jobsCollection.createIndex({ "jobId": 1 }, { name: "jobIdUnique", unique: true });
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
logger.info("Setting up indexes for 'files' collection...");
|
|
98
|
+
const filesIndexes = await filesCollection.indexes();
|
|
99
|
+
|
|
100
|
+
// Index composé pour les lookups fréquents par GUID et utilisateur
|
|
101
|
+
const compoundGuidUserIndexName = 'file_guid_user_idx';
|
|
102
|
+
if (!filesIndexes.find(i => i.name === compoundGuidUserIndexName)) {
|
|
103
|
+
await filesCollection.createIndex({ guid: 1, user: 1 }, { name: compoundGuidUserIndexName });
|
|
104
|
+
logger.info(`Created compound index '${compoundGuidUserIndexName}' on 'files' collection (guid: 1, user: 1).`);
|
|
105
|
+
} else {
|
|
106
|
+
logger.info(`Index '${compoundGuidUserIndexName}' already exists on 'files' collection.`);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const uniqueGuidIndexName = 'file_guid_unique_idx';
|
|
110
|
+
const existingGuidIndex = filesIndexes.find(i => i.name === uniqueGuidIndexName);
|
|
111
|
+
if (!existingGuidIndex) {
|
|
112
|
+
await filesCollection.createIndex({ guid: 1 }, { name: uniqueGuidIndexName, unique: true });
|
|
113
|
+
logger.info(`Created unique index '${uniqueGuidIndexName}' on 'guid' for 'files' collection.`);
|
|
114
|
+
} else if (existingGuidIndex.name !== uniqueGuidIndexName || !existingGuidIndex.unique) {
|
|
115
|
+
logger.warn(`An index on 'guid' exists for 'files' collection (name: ${existingGuidIndex.name}, unique: ${existingGuidIndex.unique}), but not matching desired spec (name: ${uniqueGuidIndexName}, unique: true). Manual review might be needed.`);
|
|
116
|
+
} else {
|
|
117
|
+
logger.info(`Unique index on 'guid' (name: '${existingGuidIndex.name}') already exists for 'files' collection.`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (! await packsCollection.indexExists("_user") ) {
|
|
121
|
+
await packsCollection.createIndex({_user: 1});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
// Create the uploads directories
|
|
126
|
+
await mkdir(path.join("uploads", "tmp"),{ recursive: true});
|
|
127
|
+
|
|
128
|
+
}else {
|
|
129
|
+
modelsCollection = getCollection("models");
|
|
130
|
+
datasCollection = getCollection(Config.Get('dataCollection','datas'));
|
|
131
|
+
filesCollection = getCollection("files");
|
|
132
|
+
packsCollection = getCollection("packs");
|
|
133
|
+
historyCollection = getCollection("history");
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Sub modules
|
|
137
|
+
backupInit(defaultEngine);
|
|
138
|
+
historyInit(defaultEngine);
|
|
139
|
+
validationInit(defaultEngine);
|
|
140
|
+
relationsInit(defaultEngine);
|
|
141
|
+
scheduleInit(defaultEngine);
|
|
142
|
+
userInit(defaultEngine);
|
|
143
|
+
operationsInit(defaultEngine);
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
await registerRoutes(engine);
|
|
147
|
+
logger = engine.getComponent(Logger);
|
|
148
|
+
|
|
149
|
+
// set backup scheduler
|
|
150
|
+
schedule.scheduleJob("0 2 * * *", jobDumpUserData);
|
|
151
|
+
//await jobDumpUserData();
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
schedule.scheduleJob("0 0 * * *", async () => {
|
|
155
|
+
const dt = new Date();
|
|
156
|
+
dt.setTime(dt.getTime()-1000*3600*24*14);
|
|
157
|
+
await deleteData("request", {"$lt": ["$timestamp",dt.toISOString()]}, null, false);
|
|
158
|
+
});
|
|
159
|
+
await scheduleAlerts();
|
|
160
|
+
|
|
161
|
+
// Triggers
|
|
162
|
+
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Vérifie si l'ajout de nouvelles données dépasserait la capacité de stockage globale du serveur.
|
|
168
|
+
* @param {number} incomingDataSize - La taille des données entrantes en octets.
|
|
169
|
+
* @returns {Promise<{isSufficient: boolean, free?: number, total?: number, error?: string}>}
|
|
170
|
+
*/
|
|
171
|
+
export async function checkServerCapacity(incomingDataSize = 0) {
|
|
172
|
+
try {
|
|
173
|
+
const diskSpace = await checkDiskSpace(DATA_STORAGE_PATH);
|
|
174
|
+
const { free, size } = diskSpace;
|
|
175
|
+
|
|
176
|
+
const storageMargin = Config.Get('storageSafetyMargin', storageSafetyMargin);
|
|
177
|
+
// Limite maximale d'utilisation du disque (ex: 90% de la taille totale)
|
|
178
|
+
const maxAllowedUsage = size * storageMargin;
|
|
179
|
+
const currentUsage = size - free;
|
|
180
|
+
const projectedUsage = currentUsage + incomingDataSize;
|
|
181
|
+
|
|
182
|
+
if (projectedUsage > maxAllowedUsage) {
|
|
183
|
+
logger.warn(`[checkServerCapacity] Alert: Projected usage (${projectedUsage} bytes) would exceed the server's safety limit (${maxAllowedUsage} bytes).`);
|
|
184
|
+
return {
|
|
185
|
+
isSufficient: false,
|
|
186
|
+
free,
|
|
187
|
+
total: size
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
return { isSufficient: true, free, total: size };
|
|
191
|
+
} catch (err) {
|
|
192
|
+
logger.error(`[checkServerCapacity] CRITICAL: Failed to check disk space: ${err.message}. Allowing write operation as a failsafe. Please investigate disk permissions or configuration.`);
|
|
193
|
+
// Failsafe: On autorise l'ure si la vérification échoue, mais on logue une erreur critique.
|
|
194
|
+
return { isSufficient: true, error: 'Could not verify disk space.' };
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
export const getResource = async (guid, user) => {
|
|
200
|
+
if (!guid) throw new Error("Le GUID du fichier est requis.");
|
|
201
|
+
if (!isGUID(guid)) throw new Error("Le GUID du fichier n'est pas valide.");
|
|
202
|
+
|
|
203
|
+
const collection = getCollection("files");
|
|
204
|
+
const file = await collection.findOne({ guid });
|
|
205
|
+
|
|
206
|
+
if (!file) {
|
|
207
|
+
throw new Error("Fichier non trouvé.");
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// La vérification des permissions reste la même...
|
|
211
|
+
if (user.username !== 'demo' && isLocalUser(user) && !await hasPermission(["API_ADMIN", "API_READ_FILE", `API_READ_FILE_privateFile_${guid}`], user)) {
|
|
212
|
+
if (file.user !== (user._user || user.username)) {
|
|
213
|
+
throw new Error("Vous n'êtes pas autorisé à accéder à ce fichier.");
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// On retourne des informations différentes selon le type de stockage
|
|
218
|
+
if (file.storage === 's3') {
|
|
219
|
+
return {
|
|
220
|
+
success: true,
|
|
221
|
+
storage: 's3',
|
|
222
|
+
s3Key: file.filename, // 'filename' contient la clé S3
|
|
223
|
+
mimeType: file.mimeType
|
|
224
|
+
// Idlement, on aurait aussi le nom de fichier original ici
|
|
225
|
+
};
|
|
226
|
+
} else { // Par défaut, on considère le stockage local
|
|
227
|
+
// --- CORRECTIF DE SÉCURITÉ ---
|
|
228
|
+
// Ne jamais faire confiance au chemin stocké en base de données.
|
|
229
|
+
// Toujours reconstruire le chemin à partir d'éléments fiables.
|
|
230
|
+
const uploadDir = path.join(process.cwd(), "uploads", "private");
|
|
231
|
+
const safeFilepath = path.join(uploadDir, file.filename); // file.filename est le nom sécurisé (guid.ext)
|
|
232
|
+
|
|
233
|
+
// Vérification supplémentaire pour s'assurer que le chemin résolu est bien dans le répertoire attendu.
|
|
234
|
+
if (!safeFilepath.startsWith(uploadDir)) {
|
|
235
|
+
throw new Error("Tentative d'accès à un chemin non autorisé.");
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (!fs.existsSync(safeFilepath)) {
|
|
239
|
+
throw new Error("Fichier non trouvé sur le serveur.");
|
|
240
|
+
}
|
|
241
|
+
return {
|
|
242
|
+
success: true,
|
|
243
|
+
storage: 'local',
|
|
244
|
+
filepath: safeFilepath, // On retourne le chemin sécurisé et reconstruit
|
|
245
|
+
filename: file.filename,
|
|
246
|
+
mimeType: file.mimeType
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
export async function handleDemoInitialization(req, res) {
|
|
253
|
+
const user = req.me;
|
|
254
|
+
const body = req.fields;
|
|
255
|
+
const packs = body.packs;
|
|
256
|
+
const models = (Object.keys(profiles).includes(body.profile) && profiles[body.profile].models) || '';
|
|
257
|
+
if (!isDemoUser(user)) {
|
|
258
|
+
return res.status(403).json({ success: false, error: "This action is only for demo users." });
|
|
259
|
+
}
|
|
260
|
+
if (!Array.isArray(models) || models.length === 0) {
|
|
261
|
+
return res.status(400).json({ success: false, error: "A valid 'models' array is required." });
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
logger.info(`[Demo Init] Starting initialization for user '${user.username}' with ${models.length} models.`);
|
|
265
|
+
|
|
266
|
+
try {
|
|
267
|
+
// 1. Nettoyage de l'environnement (inchangé)
|
|
268
|
+
const datasCollection = getCollection(Config.Get('dataCollection',"datas"));
|
|
269
|
+
const modelsCollection = getCollection("models");
|
|
270
|
+
const filesCollection = getCollection("files");
|
|
271
|
+
|
|
272
|
+
await datasCollection.deleteMany({ _user: user.username });
|
|
273
|
+
await modelsCollection.deleteMany({ _user: user.username });
|
|
274
|
+
const files = await filesCollection.find({ user: user.username }).toArray();
|
|
275
|
+
for (const file of files) {
|
|
276
|
+
await removeFile(file.guid, user).catch(e => logger.error(e.message));
|
|
277
|
+
}
|
|
278
|
+
await cancelAlerts(user);
|
|
279
|
+
logger.info(`[Demo Init] Environment cleaned for user '${user.username}'.`);
|
|
280
|
+
|
|
281
|
+
const packToInstall = {
|
|
282
|
+
name: `dynamic-pack-for-${user.username}-${Date.now()}`,
|
|
283
|
+
description: `Dynamically generated pack for profile models.`,
|
|
284
|
+
models: models,
|
|
285
|
+
data: {}
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
logger.info(`[Demo Init] Installing dynamically generated pack with models: [${models.join(', ')}].`);
|
|
289
|
+
|
|
290
|
+
// Create and install pack
|
|
291
|
+
const result = await installPack(packToInstall, user, req.query.lang || 'en');
|
|
292
|
+
|
|
293
|
+
await sequential(packs.map(p => {
|
|
294
|
+
return () => installPack(p, user, req.query.lang || 'en');
|
|
295
|
+
}));
|
|
296
|
+
|
|
297
|
+
if (result.success || result.modifiedCount > 0) {
|
|
298
|
+
|
|
299
|
+
await Event.Trigger('OnDemoUserAdded', "event", "system", req.me.username);
|
|
300
|
+
logger.info(`[Demo Init] Pack installed successfully for user '${user.username}'.`);
|
|
301
|
+
res.status(200).json({ success: true, message: "Demo environment initialized successfully.", summary: result.summary });
|
|
302
|
+
} else {
|
|
303
|
+
logger.error(`[Demo Init] Pack installation failed for user '${user.username}'.`);
|
|
304
|
+
res.status(200).json({ success: false, error: 'Demo pack installation failed.', errors: result.errors });
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
} catch (error) {
|
|
308
|
+
logger.error(`[Demo Init] Critical error during initialization for user '${user.username}':`, error);
|
|
309
|
+
res.status(500).json({ success: false, error: 'An internal server error occurred during initialization.' });
|
|
310
|
+
}
|
|
311
|
+
}
|