data-primals-engine 1.2.6 → 1.3.1

package/src/modules/data/data.js

@@ -1,35 +1,25 @@
 import {Logger} from "../../gameObject.js";
 import {BSON, ObjectId} from "mongodb";
-import util, {promisify} from 'node:util';
+import {promisify} from 'node:util';
 import crypto from "node:crypto";
-import {exec, execFile} from 'node:child_process';
+import {execFile} from 'node:child_process';
 import sanitizeHtml from 'sanitize-html';
 import * as tar from "tar";
 import process from "node:process";
 import {randomColor} from "randomcolor";
 import cronstrue from 'cronstrue/i18n.js';
 import {mkdir} from 'node:fs/promises';
-import {
-    anonymizeText,
-    getDefaultForType,
-    getFieldValueHash,
-    getUserId,
-    isDemoUser,
-    isLocalUser
-} from "../../data.js";
+import {onInit as historyInit} from "./data.history.js";
+import {anonymizeText, getDefaultForType, getFieldValueHash, getUserId, isDemoUser, isLocalUser} from "../../data.js";
 import {
     allowedFields,
     dbName,
     install,
     maxAlertsPerUser,
-    maxBytesPerSecondThrottleData,
     maxExportCount,
     maxFileSize,
     maxFilterDepth,
-    maxMagnetsDataPerModel,
-    maxMagnetsModels,
     maxModelNameLength,
-    maxModelsPerUser,
     maxPasswordLength,
     maxPostData,
     maxRelationsPerData,
@@ -43,6 +33,7 @@ import {
     storageSafetyMargin
 } from "../../constants.js";
 import {
+    createCollection,
     getCollection,
     getCollectionForUser,
     getUserCollectionName,
@@ -52,7 +43,7 @@ import {
 } from "../mongodb.js";
 import {dbUrl, MongoClient, MongoDatabase} from "../../engine.js";
 import path from "node:path";
-import {getFileExtension, getObjectHash, getRandom, isGUID, isPlainObject, randomDate, sleep, uuidv4} from "../../core.js";
+import {getObjectHash, getRandom, isGUID, isPlainObject, randomDate} from "../../core.js";
 import {Event} from "../../events.js";
 import fs from "node:fs";
 import schedule from "node-schedule";
@@ -67,12 +58,9 @@ import {
 import NodeCache from "node-cache";
 import AWS from 'aws-sdk';
 import checkDiskSpace from "check-disk-space";
-import {addFile,  removeFile} from "../file.js";
+import {addFile, removeFile} from "../file.js";
 import {downloadFromS3, getUserS3Config, listS3Backups, uploadToS3} from "../bucket.js";
-import {
-    calculateTotalUserStorageUsage,
-    hasPermission
-} from "../user.js";
+import {calculateTotalUserStorageUsage, hasPermission, middlewareAuthenticator} from "../user.js";
 import {getAllPacks} from "../../packs.js";
 import {Config} from "../../config.js";
 import {profiles} from "../../../client/src/constants.js";
@@ -193,7 +181,7 @@ export const dataTypes = {
             }
             return value;
         },
-        anonymize: anonymizeText
+        anonymize: (str) => anonymizeText(typeof(str) ==='object' ? JSON.stringify(str): str)
     },
     richtext: {
         validate: (value, field) => {
@@ -500,53 +488,8 @@ export const getAPILang = (langs) => {
 }
 
 
-export function validateModelStructure(modelStructure) {
-
-    const objectKeys = Object.keys(modelStructure);
-    if( objectKeys.find(o => !["name", "_user", "locked", "_id", "description", "maxRequestData", "fields"].includes(o)) ){
-        throw new Error(i18n.t('api.model.invalidStructure'));
-    }
-
-    // Vérification du type de name
-    if (typeof modelStructure.name !== 'string' || !modelStructure.name) {
-        throw new Error(i18n.t("api.validate.requiredFieldString", ["name"]));
-    }
-
-    // Vérification du type de description
-    if (typeof modelStructure.description !== 'string') {
-        throw new Error(i18n.t("api.validate.fieldString", ["description"]));
-    }
-
-    // Vérification de la présence et du type du tableau fields
-    if (!Array.isArray(modelStructure.fields)) {
-        throw new Error(i18n.t('api.validate.fieldArray', ["fields"]));
-    }
-
-    // Vérification de chaque champ dans le tableau fields
-    for (const field of modelStructure.fields) {
-        validateField(field);
-    }
-
-    if (modelStructure.constraints) {
-        if (!Array.isArray(modelStructure.constraints)) {
-            throw new Error('Model "constraints" property must be an array.');
-        }
-        const fieldNames = new Set(modelStructure.fields.map(f => f.name));
-        for (const constraint of modelStructure.constraints) {
-            if (constraint.type === 'unique') {
-                if (!constraint.name || !Array.isArray(constraint.keys) || constraint.keys.length === 0) {
-                    throw new Error('Unique constraint must have a "name" and a non-empty "keys" array.');
-                }
-                for (const key of constraint.keys) {
-                    if (!fieldNames.has(key)) {
-                        throw new Error(`Constraint key "${key}" in constraint "${constraint.name}" does not exist as a field in the model.`);
-                    }
-                }
-            }
-        }
-    }
-
-    return true; // La structure du modèle est valide
+export async function validateModelStructure(modelStructure) {
+    return await Event.Trigger("OnValidateModelStructure", "event", "system", modelStructure);
 }
 
 const validateField = (field) => {
@@ -905,7 +848,7 @@ async function runStatefulAlertJob(alertId) {
                 timestamp: new Date().toISOString(),
                 message: `Alerte '${alertDoc.name}': ${count} élément(s) correspondent à votre condition.`
             };
-            sendSseToUser(alertDoc._user, alertPayload);
+            await sendSseToUser(alertDoc._user, alertPayload);
 
             // Update state in DB to prevent re-notification
             await datasCollection.updateOne(
@@ -1030,7 +973,7 @@ export const editModel = async (user, id, data) => {
     const dataModel = data;
     try {
         const collection = await getCollectionForUser(user);
-        validateModelStructure(dataModel);
+        await validateModelStructure(dataModel);
 
         const el = await modelsCollection.findOne({ $and: [
             {_user: {$exists: true}},
@@ -1119,8 +1062,8 @@ export const editModel = async (user, id, data) => {
 
         const newModel = await modelsCollection.findOne({_id : oid});
         const res = ({ success: true, data: newModel });
-        const plugin = Event.Trigger("OnModelEdited", "event", "system", engine, newModel);
-        Event.Trigger("OnModelEdited", "event", "user", plugin?.data || newModel);
+        const plugin = await Event.Trigger("OnModelEdited", "event", "system", engine, newModel);
+        await Event.Trigger("OnModelEdited", "event", "user", plugin?.data || newModel);
         return plugin || res
     } catch (e) {
         logger.error(e);
@@ -1129,38 +1072,71 @@ export const editModel = async (user, id, data) => {
 };
 
 
-export async function handleCustomEndpointRequest(req, res) {
+export async function middlewareEndpointAuthenticator(req, res, next) {
     const { path } = req.params;
     const method = req.method.toUpperCase();
-
-    const user = req.me;
-    if (!user) {
-        return res.status(401).json({ success: false, message: 'Authentication required.' });
-    }
+    const datasCollection = getCollection('datas');
 
     try {
-        // 1. Trouver l'endpoint correspondant dans la base de données
-        const endpointSearch = await searchData({
-            model: 'endpoint',
-            filter: {
-                path: path,
-                method: method,
-                isActive: true
-            },
-            limit: 1
-        }, user);
+        const endpointDef = await datasCollection.findOne({
+            _model: 'endpoint',
+            path: path,
+            method: method,
+            isActive: true
+        });
 
-        if (endpointSearch.count === 0) {
-            logger.warn(`[Endpoint] 404 - No active endpoint found for user '${user.username}', path '${path}', method '${method}'.`);
+        if (!endpointDef) {
             return res.status(404).json({ success: false, message: 'Endpoint not found.' });
         }
 
-        const endpointDef = endpointSearch.data[0];
+        // Attacher la définition à la requête pour que le handler suivant puisse l'utiliser
+        req.endpointDef = endpointDef;
+
+        // Si l'endpoint n'est PAS public, on exécute le vrai middleware d'authentification
+        if (!endpointDef.isPublic) {
+            // On "chaîne" vers le middleware authenticator standard.
+            // Il se chargera de vérifier le token et de renvoyer une 401 si nécessaire.
+            return middlewareAuthenticator(req, res, next);
+        }
+
+        // Si l'endpoint EST public, on passe simplement à la suite.
+        next();
+
+    } catch (error) {
+        logger.error(`[EndpointAuth] Critical error: ${error.message}`, error.stack);
+        res.status(500).json({ success: false, message: 'Internal server error during endpoint authentication.' });
+    }
+}
+
+export async function handleCustomEndpointRequest(req, res) {
+    const endpointDef = req.endpointDef;
+
+    try {
+        let executionUser = null;
+
+        // 1. Déterminer le contexte utilisateur pour l'exécution
+        if (endpointDef.isPublic) {
+            // Pour les endpoints publics, on exécute le script en tant que propriétaire de l'endpoint.
+            if (!endpointDef._user) {
+                logger.error(`[Endpoint] Misconfiguration: Public endpoint '${endpointDef.name}' (ID: ${endpointDef._id}) has no owner.`);
+                return res.status(500).json({ success: false, message: 'Endpoint misconfigured: owner missing.' });
+            }
+            executionUser = await engine.userProvider.findUserByUsername(endpointDef._user);
+            if (!executionUser) {
+                logger.error(`[Endpoint] Execution failed: Owner '${endpointDef._user}' for public endpoint '${endpointDef.name}' not found.`);
+                return res.status(500).json({ success: false, message: 'Endpoint owner not found.' });
+            }
+            logger.info(`[Endpoint] Public endpoint '${endpointDef.name}' running as owner '${executionUser.username}'.`);
+        } else {
+            // Pour les endpoints privés, l'utilisateur a déjà été authentifié par le middleware.
+            // req.me est garanti d'exister ici.
+            executionUser = req.me;
+            logger.info(`[Endpoint] Private endpoint '${endpointDef.name}' running as authenticated user '${executionUser.username}'.`);
+        }
 
         // 2. Préparer le contexte pour le script
-        // On donne au script accès au corps, aux paramètres de la requête, etc.
         const contextData = {
-            request:{
+            request: {
                 body: req.fields,
                 query: req.query,
                 params: req.params,
@@ -1168,34 +1144,29 @@ export async function handleCustomEndpointRequest(req, res) {
             }
         };
 
-        // 3. Exécuter le code de l'endpoint en utilisant notre sandbox sécurisé
-        logger.info(`[Endpoint] Executing endpoint '${endpointDef.name}' for user '${user.username}'.`);
+        // 3. Exécuter le code de l'endpoint
         const result = await executeSafeJavascript(
-            { script: endpointDef.code }, // On passe la définition du script
+            { script: endpointDef.code },
             contextData,
-            user
+            executionUser // Use the determined user for execution
         );
 
         // 4. Envoyer la réponse
         if (result.success) {
-            // Le script a réussi, on retourne sa sortie
             res.status(200).json(result.data);
         } else {
-            // Le script a échoué, on retourne une erreur 500 avec les logs
             logger.error(`[Endpoint] Execution failed for '${endpointDef.name}'. Error: ${result.message}`);
-
-            const r = {
+            const responseError = {
                 success: false,
                 message: 'Endpoint script execution failed.',
-                // On peut choisir d'exposer les logs pour le débogage
-                details: result.message
+                details: result.message,
+                logs: result.logs
             };
-            r.logs = result.logs;
-            res.status(500).json(r);
+            res.status(500).json(responseError);
         }
 
     } catch (error) {
-        logger.error(`[Endpoint] Critical error handling request for path '${path}': ${error.message}`, process.env.NODE_ENV === 'development'? error.stack : error.stack[0]);
+        logger.error(`[Endpoint] Critical error handling request for path '${endpointDef.path}': ${error.message}`, error.stack);
         res.status(500).json({ success: false, message: 'An internal server error occurred.' });
     }
 }
@@ -1206,12 +1177,13 @@ export async function onInit(defaultEngine) {
 
     engine.use(middleware({ whitelist: mongoDBWhitelist }));
 
-    let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection;
+    let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection, historyCollection;
 
     if( install ) {
-        datasCollection = await MongoDatabase.createCollection("datas");
-        filesCollection = await MongoDatabase.createCollection("files");
-        packsCollection = await MongoDatabase.createCollection("packs");
+        datasCollection = await createCollection("datas");
+        historyCollection = await createCollection("history");
+        filesCollection = await createCollection("files");
+        packsCollection = await createCollection("packs");
         //data
         const indexes = await datasCollection.indexes();
         if (!indexes.find(i => i.name === 'genericPartialIndex')) {
@@ -1237,7 +1209,7 @@ export async function onInit(defaultEngine) {
             await datasCollection.createIndex({_model: 1, _user: 1}, { name: 'modelUserIndex'});
         }
 
-        const jobsCollection = await MongoDatabase.createCollection("job_locks");
+        const jobsCollection = await createCollection("job_locks");
         if (! await jobsCollection.indexExists("jobTTLIndex") ) {
             await jobsCollection.createIndex({ "lockedUntil": 1 }, { name: "jobTTLIndex", expireAfterSeconds: 0 });
         }
@@ -1281,6 +1253,7 @@ export async function onInit(defaultEngine) {
         datasCollection = getCollection("datas");
         filesCollection = getCollection("files");
         packsCollection = getCollection("packs");
+        historyCollection = getCollection("history");
     }
     await registerRoutes(engine);
     logger = engine.getComponent(Logger);
@@ -1297,6 +1270,61 @@ export async function onInit(defaultEngine) {
     });
     await scheduleAlerts();
 
+    // Triggers
+
+    Event.Listen("OnValidateModelStructure", async (modelStructure) =>{
+
+        const objectKeys = Object.keys(modelStructure);
+
+        if( objectKeys.find(o => !["name", "_user", "history", "locked", "_id", "description", "maxRequestData", "fields"].includes(o)) ){
+            throw new Error(i18n.t('api.model.invalidStructure'));
+        }
+
+        // Vérification du type de name
+        if (typeof modelStructure.name !== 'string' || !modelStructure.name) {
+            throw new Error(i18n.t("api.validate.requiredFieldString", ["name"]));
+        }
+
+        // Vérification du type de description
+        if (typeof modelStructure.description !== 'string') {
+            throw new Error(i18n.t("api.validate.fieldString", ["description"]));
+        }
+
+        // Vérification de la présence et du type du tableau fields
+        if (!Array.isArray(modelStructure.fields)) {
+            throw new Error(i18n.t('api.validate.fieldArray', ["fields"]));
+        }
+
+        // Vérification de chaque champ dans le tableau fields
+        for (const field of modelStructure.fields) {
+            validateField(field);
+        }
+
+        if (modelStructure.constraints) {
+            if (!Array.isArray(modelStructure.constraints)) {
+                throw new Error('Model "constraints" property must be an array.');
+            }
+            const fieldNames = new Set(modelStructure.fields.map(f => f.name));
+            for (const constraint of modelStructure.constraints) {
+                if (constraint.type === 'unique') {
+                    if (!constraint.name || !Array.isArray(constraint.keys) || constraint.keys.length === 0) {
+                        throw new Error('Unique constraint must have a "name" and a non-empty "keys" array.');
+                    }
+                    for (const key of constraint.keys) {
+                        if (!fieldNames.has(key)) {
+                            throw new Error(`Constraint key "${key}" in constraint "${constraint.name}" does not exist as a field in the model.`);
+                        }
+                    }
+                }
+            }
+        }
+
+        return true; // La structure du modèle est valide
+    }, "event", "system");
+
+
+    // Sub modules
+    historyInit(defaultEngine);
 }
 
 export const createModel = async (data) => {
@@ -1505,7 +1533,7 @@ export const insertData = async (modelName, data, files, user, triggerWorkflow =
                                     };
 
                                     // Envoyer l'alerte à l'utilisateur spécifique via SSE
-                                    const sent = sendSseToUser(doc._user, alertPayload);
+                                    const sent = await sendSseToUser(doc._user, alertPayload);
 
                                     if (sent) {
                                         logger.info(`[Scheduled Job] Successfully sent SSE alert for job ${jobId} to user ${doc._user}.`);
@@ -1546,10 +1574,18 @@ export const insertData = async (modelName, data, files, user, triggerWorkflow =
             // Attendre que toutes les opérations post-insertion (workflows, planification) soient tentées
             await Promise.allSettled(postInsertionPromises);
         }
-        const res = { success: true, insertedIds: insertedIds.map(id => id.toString()) }; // Convertir les IDs en string pour la réponse
-        const plugin = Event.Trigger("OnDataAdded", "event", "system", engine, insertedIds);
-        Event.Trigger("OnDataAdded", "event", "user", plugin?.insertedIds || insertedIds);
-        return plugin || res;
+
+        // System specific event
+        const eventPayload = { modelName, insertedIds, user };
+        await Event.Trigger("OnDataAdded", "event", "system", engine, eventPayload);
+
+        // User specific event
+        const userPayload = {...eventPayload};
+        delete userPayload['user'];
+        await Event.Trigger("OnDataAdded", "event", "user", userPayload);
+
+        // Return valid result
+        return { success: true, insertedIds: insertedIds.map(id => id.toString()) }; // Convertir les IDs en string pour la réponse
 
     } catch (error) { // Attrape les erreurs de permission ou de pushDataUnsecure
         logger.error(`[insertData] Main error during insertion process for model ${modelName}: ${error.message}`, error.stack);
@@ -1689,7 +1725,7 @@ async function initializeAndValidate(data, modelName, me) {
 
     const model = await getModel(modelName, me);
     const collection = await getCollectionForUser(me);
-    validateModelStructure(model);
+    await validateModelStructure(model);
 
     return { datas, model, collection };
 }
@@ -1752,7 +1788,7 @@ async function processDocuments(datas, model, collection, me) {
     const idMap = new Map();
     const allInsertedIds = [];
 
-    const realData = Event.Trigger("OnDataInsert", "event", "system", datas) || datas;
+    const realData = await Event.Trigger("OnDataInsert", "event", "system", datas) || datas;
     for (const doc of realData) {
         try {
             const newDocId = await insertAndResolveRelations(doc, model, collection, me, idMap);
@@ -1958,7 +1994,7 @@ async function applyFieldFilters(docToProcess, model) {
                 docToProcess[field.name],
                 field
             );
-            const realFilter = Event.Trigger('OnDataFilter', "event", "system",filter, field, docToProcess );
+            const realFilter = await Event.Trigger('OnDataFilter', "event", "system",filter, field, docToProcess );
             docToProcess[field.name] = realFilter || filter;
         }
     }
@@ -1968,7 +2004,7 @@ async function applyFieldFilters(docToProcess, model) {
 /**
  * Valide la structure et le contenu du document selon le modèle
  */
-function validateModelData(doc, model, isPatch = false) {
+async function validateModelData(doc, model, isPatch = false) {
     if (!isPatch) {
         model.fields.forEach(field => {
             const value = doc[field.name];
@@ -1990,7 +2026,7 @@ function validateModelData(doc, model, isPatch = false) {
 
         const validator = dataTypes[fieldDef.type]?.validate;
         const valid = validator && validator(value, fieldDef);
-        const realValidation = Event.Trigger('OnDataValidate', "event", "system", value, fieldDef,doc );
+        const realValidation = await Event.Trigger('OnDataValidate', "event", "system", value, fieldDef,doc );
         if (!(valid || realValidation)) {
             throw new Error(i18n.t('api.field.validationFailed', { field: fieldName, value }));
         }
@@ -2027,9 +2063,9 @@ async function insertAndResolveRelations(doc, model, collection, me, idMap) {
         docToProcess._id = new ObjectId(originalId);
     }
 
-    validateModelData(docToProcess, model);
+    await validateModelData(docToProcess, model);
     await processRelations(docToProcess, model, collection, me, idMap);
-    validateModelData(docToProcess, model);
+    await validateModelData(docToProcess, model);
     await applyFieldFilters(docToProcess, model);
     await checkUniqueFields(docToProcess, model, collection);
 
@@ -2271,9 +2307,9 @@ const internalEditOrPatchData = async (modelName, filter, data, files, user, isP
         // 4. Validation adaptée pour patch ou edit (inchangé)
         if (!isPatch) {
             const dataToValidate = { ...existingDocs[0], ...updateData };
-            validateModelData(dataToValidate, model, false);
+            await validateModelData(dataToValidate, model, false);
         } else {
-            validateModelData(updateData, model, true);
+            await validateModelData(updateData, model, true);
         }
 
         // 5. Vérification des champs uniques (inchangé)
@@ -2374,6 +2410,17 @@ const internalEditOrPatchData = async (modelName, filter, data, files, user, isP
         const bulkResult = await collection.bulkWrite(bulkOps);
         const modifiedCount = bulkResult.modifiedCount || 0;
 
+        // Déclencher l'événement OnDataEdited avec les états avant/après
+        if (modifiedCount > 0) {
+            const updatedDocs = await collection.find({ _id: { $in: ids } }).toArray();
+            await Event.Trigger("OnDataEdited", "event", "system", engine, {
+                modelName,
+                user,
+                before: existingDocs, // Documents avant la modification
+                after: updatedDocs     // Documents après la modification
+            });
+        }
+
         // 11. Tâches post-mise à jour (schedules, workflows) (inchangé)
         if (["workflowTrigger", "alert"].includes(modelName)) {
             await handleScheduledJobs(modelName, existingDocs, collection, finalDataForSet);
@@ -2434,7 +2481,7 @@ async function handleScheduledJobs(modelName, existingDocs, collection, updateDa
             schedule.scheduleJob(jobId, updatedDoc.cronExpression, async () => {
                 logger.info(`[Scheduled Job] Cron triggered for job ${jobId}`);
                 await runScheduledJobWithDbLock(jobId, async () => {
-                    sendSseToUser(updatedDoc._user, {
+                    await sendSseToUser(updatedDoc._user, {
                         type: 'cron_alert',
                         triggerId: updatedDoc._id.toString(),
                         triggerName: updatedDoc.name,
@@ -2657,8 +2704,8 @@ export const deleteData = async (modelName, filter, user ={}, triggerWorkflow, w
         }
 
         const res = { success: true, deletedCount }
-        const plugin = Event.Trigger("OnDataDeleted", "event", "system", engine, {model:modelName, filter});
-        Event.Trigger("OnDataDeleted", "event", "user", {model:modelName, filter});
+        const plugin = await Event.Trigger("OnDataDeleted", "event", "system", engine, {model:modelName, filter});
+        await Event.Trigger("OnDataDeleted", "event", "user", {model:modelName, filter});
         return plugin || res;
 
     } catch (error) {
@@ -3166,8 +3213,8 @@ export const searchData = async (query, user) => {
     data = await handleFields(modelElement, data, user);
 
     const res = {data, count: count[0]?.count || 0};
-    const plugin = Event.Trigger("OnDataSearched", "event", "system", engine, {data, count: count[0]?.count});
-    Event.Trigger("OnDataSearched", "event", "user", plugin || {data, count: count[0]?.count});
+    const plugin = await Event.Trigger("OnDataSearched", "event", "system", engine, {data, count: count[0]?.count});
+    await Event.Trigger("OnDataSearched", "event", "user", plugin || {data, count: count[0]?.count});
     return plugin || res;
 }
 
@@ -3244,7 +3291,7 @@ export const importData = async(options, files, user) => {
                                 });
                             }
 
-                            validateModelStructure(modelData);
+                            await validateModelStructure(modelData);
                             await modelsCollection.insertOne(modelData);
                             logger.info(`[Model Import] Successfully imported model '${modelName}' for user ${user.username}.`);
                         } catch (e) {
@@ -3326,7 +3373,7 @@ export const importData = async(options, files, user) => {
                                 if (insertedIdsArray && insertedIdsArray.length > 0) {
                                     importJobs[importJobId].processedRecords += insertedIdsArray.length;
                                     logger.debug(`[Import Job ${importJobId}] Processed chunk for '${modelName}': ${insertedIdsArray.length} records. Total processed: ${importJobs[importJobId].processedRecords}`);
-                                    sendSseToUser(user.username, {
+                                    await sendSseToUser(user.username, {
                                         type: 'import_progress',
                                         job: importJobs[importJobId]
                                     });
@@ -3430,7 +3477,7 @@ export const importData = async(options, files, user) => {
                                     const insertedIdsArray = await pushDataUnsecure(chunk, modelNameForImport, user, {});
                                     if (insertedIdsArray && insertedIdsArray.length > 0) {
                                         importJobs[importJobId].processedRecords += insertedIdsArray.length;
-                                        sendSseToUser(user.username, {
+                                        await sendSseToUser(user.username, {
                                             type: 'import_progress',
                                             job: importJobs[importJobId]
                                         });
@@ -3524,7 +3571,7 @@ export const importData = async(options, files, user) => {
                                     const insertedIdsArray = await pushDataUnsecure(chunk, modelNameForImport, user, {});
                                     if (insertedIdsArray && insertedIdsArray.length > 0) {
                                         importJobs[importJobId].processedRecords += insertedIdsArray.length;
-                                        sendSseToUser(user.username, {
+                                        await sendSseToUser(user.username, {
                                             type: 'import_progress',
                                             job: importJobs[importJobId]
                                         });
@@ -3569,7 +3616,7 @@ export const importData = async(options, files, user) => {
                 } else {
                     importJobs[importJobId].status = 'completed';
                 }
-                sendSseToUser(user.username, {
+                await sendSseToUser(user.username, {
                     type: 'import_progress',
                     job: importJobs[importJobId]
                 });
@@ -3705,8 +3752,8 @@ export const exportData= async (options, user) =>{
     }
 
     const res = { success: true, data: exportResults, models: modelsToExport };
-    const plugin = Event.Trigger("OnDataExported", "event", "system", engine, exportResults, modelsToExport);
-    Event.Trigger("OnDataExported", "event", "user", plugin?.exportResults || exportResults, plugin?.modelsToExport || modelsToExport);
+    const plugin = await Event.Trigger("OnDataExported", "event", "system", engine, exportResults, modelsToExport);
+    await Event.Trigger("OnDataExported", "event", "user", plugin?.exportResults || exportResults, plugin?.modelsToExport || modelsToExport);
     return plugin || res;
 }
 
@@ -4058,7 +4105,7 @@ export const loadFromDump = async (user, options = {}) => {
         modelsCache.flushAll();
 
         logger.info(`[${action}] Restore successful for user ${user.username}.`);
-        Event.Trigger("OnDataRestored", "event", "system");
+        await Event.Trigger("OnDataRestored", "event", "system");
 
     } finally {
         // --- GUARANTEED CLEANUP ---
@@ -4242,68 +4289,94 @@ async function manageBackupRotation(user, backupFrequency, s3Config = null) { //
 }
 
 /**
- * Installe les modèles et les données d'un pack pour un utilisateur.
- * Gère les modèles, les données, et les relations complexes (y compris les références futures)
- * via un système en deux passes optimisé.
+ * Installs pack models and data for a user or globally.
+ * Can accept either a pack ID (to install from database) or a direct pack JSON object.
  *
- * @param {object} logger - L'instance du logger.
- * @param {string} packId - L'ID du pack à installer depuis la collection 'packs'.
- * @param {object} user - L'objet utilisateur pour qui installer le pack.
- * @param {string} lang - Le code de langue pour les données spécifiques.
+ * @param {object} logger - Logger instance
+ * @param {string|object} packIdentifier - Either pack ID (string) or pack JSON object
+ * @param {object|null} user - User object (if installing for user) or null (for global install)
+ * @param {string} [lang='en'] - Language code for localized data
  * @returns {Promise<{success: boolean, summary: object, errors: Array, modifiedCount: number}>}
  */
-export async function installPack(packId, user, lang) {
+export async function installPack(packIdentifier, user = null, lang = 'en', isTemporary= false) {
+    let pack;
     const packsCollection = getCollection('packs');
-    const pack = await packsCollection.findOne({ _id: new ObjectId(packId) });
 
-    if (!pack) {
-        throw new Error(`Pack with ID ${packId} not found.`);
+    // Determine if we're working with an ID or direct pack object
+    if (typeof packIdentifier === 'string') {
+        // Existing behavior - fetch from database
+        pack = await packsCollection.findOne({ _id: new ObjectId(packIdentifier) });
+        if (!pack) {
+            throw new Error(`Pack with ID ${packIdentifier} not found.`);
+        }
+    } else if (typeof packIdentifier === 'object' && packIdentifier !== null) {
+        // New behavior - use provided pack object directly
+        pack = packIdentifier;
+
+        // Validate basic pack structure
+        if (!pack.name || (!pack.models && !pack.data)) {
+            throw new Error('Invalid pack structure - must contain at least name and models or data');
+        }
+    } else {
+        throw new Error('Invalid pack identifier - must be either pack ID string or pack object');
     }
 
-    logger.info(`--- Starting installation of pack '${pack.name}' for user '${user.username}' ---`);
+    const username = user ? user.username : null;
+    const logPrefix = username
+        ? `Installing pack '${pack.name}' for user '${username}'`
+        : `Installing pack '${pack.name}' globally`;
+
+    logger.info(`--- ${logPrefix} ---`);
 
     const summary = {
         models: { installed: [], skipped: [], failed: [] },
         datas: { inserted: 0, updated: 0, skipped: 0, failed: 0 }
     };
     const errors = [];
-    const collection = await getCollectionForUser(user);
+    const collection = user ? await getCollectionForUser(user) : getCollection('data');
     const tempIdToNewIdMap = {};
     const linkCache = new Map();
 
-    // --- PHASE 1: Installation des Modèles ---
+    // --- PHASE 1: MODEL INSTALLATION ---
     if (Array.isArray(pack.models)) {
-        const userModels = await modelsCollection.find({ _user: user.username }).toArray();
-        const userModelNames = userModels.map(m => m.name);
+        // For user installs, check existing models
+        const existingModels = user
+            ? await modelsCollection.find({ _user: username }).toArray()
+            : await modelsCollection.find({ _user: { $exists: false } }).toArray();
+
+        const existingModelNames = existingModels.map(m => m.name);
 
         for (const modelOrName of pack.models) {
             try {
                 const modelName = typeof modelOrName === 'string' ? modelOrName : modelOrName?.name;
                 if (!modelName) throw new Error('Model definition in pack is missing a name.');
 
-                if (userModelNames.includes(modelName)) {
-                    logger.debug(`[Model Install] Skipping '${modelName}': already exists for user.`);
+                if (existingModelNames.includes(modelName)) {
+                    logger.debug(`[Model Install] Skipping '${modelName}': already exists`);
                     summary.models.skipped.push(modelName);
                     continue;
                 }
 
-                let modelToInstall;
-                if (typeof modelOrName === 'string') {
-                    const sharedModel = await modelsCollection.findOne({ name: modelName, _user: { $exists: false } });
-                    if (!sharedModel) throw new Error(`Shared model '${modelName}' not found.`);
-                    const { _id, ...sharedModelData } = sharedModel;
-                    modelToInstall = sharedModelData;
-                } else {
-                    modelToInstall = { ...modelOrName };
+                const modelToInstall = typeof modelOrName === 'string'
+                    ? await modelsCollection.findOne({ name: modelName, _user: { $exists: false } })
+                    : { ...modelOrName };
+
+                if (!modelToInstall) {
+                    throw new Error(`Model '${modelName}' not found in shared models`);
                 }
 
-                modelToInstall._user = user.username;
-                delete modelToInstall._id;
-                modelToInstall.locked = false;
-                if (modelToInstall.fields) modelToInstall.fields.forEach(f => f.locked = false);
+                // Prepare model for installation
+                const preparedModel = { ...modelToInstall };
+                if (user) preparedModel._user = username;
+                delete preparedModel._id;
+                preparedModel.locked = false;
+
+                if (preparedModel.fields) {
+                    preparedModel.fields.forEach(f => f.locked = false);
+                }
 
-                validateModelStructure(modelToInstall);
-                await modelsCollection.insertOne(modelToInstall);
+                await validateModelStructure(preparedModel);
+                await modelsCollection.insertOne(preparedModel);
                 summary.models.installed.push(modelName);
 
             } catch (e) {
@@ -4314,13 +4387,14 @@ export async function installPack(packId, user, lang) {
         }
     }
 
-    // --- PHASE 2: Installation des Données ---
-    const dataToInstall = { ...pack.data?.all, ...pack.data?.[user.lang || lang || 'en'] };
-    if (!dataToInstall || typeof dataToInstall !== 'object' || Object.keys(dataToInstall).length === 0) {
+    // --- PHASE 2: DATA INSTALLATION ---
+    const dataToInstall = { ...pack.data?.all, ...pack.data?.[lang] };
+    if (!dataToInstall || Object.keys(dataToInstall).length === 0) {
         logger.warn(`Pack '${pack.name}' has no data to install.`);
         return { success: errors.length === 0, summary, errors, modifiedCount: 0 };
     }
 
+    // Process link references (same as original)
     const linkQueue = [];
     for (const modelName in dataToInstall) {
         if (Array.isArray(dataToInstall[modelName])) {
@@ -4330,7 +4404,6 @@ export async function installPack(packId, user, lang) {
 
                 for (const fieldName in docSource) {
                     if (isPlainObject(docSource[fieldName]) && docSource[fieldName].$link) {
-                        // CORRECTION 1: On ajoute le nom du modèle source à la file d'attente
                         linkQueue.push({
                             sourceTempId: tempId,
                             sourceModelName: modelName,
@@ -4343,21 +4416,21 @@ export async function installPack(packId, user, lang) {
         }
     }
 
-    // --- PASSE 1: INSERTION PAR LOT ---
-    logger.info("[Pack Install] Starting Pass 1: Batch Insertion & ID Mapping.");
+    // --- PASS 1: BATCH INSERTION ---
+    logger.info("[Pack Install] Starting Pass 1: Batch Insertion & ID Mapping");
     for (const modelName in dataToInstall) {
-        if (!Object.prototype.hasOwnProperty.call(dataToInstall, modelName) || !Array.isArray(dataToInstall[modelName])) continue;
+        if (!Array.isArray(dataToInstall[modelName])) continue;
 
         const documents = dataToInstall[modelName];
         if (documents.length === 0) continue;
 
         const docsToInsert = [];
-
         const modelDefForHash = await getModel(modelName, user);
 
         for (const docSource of documents) {
             let docForInsert = { ...docSource };
 
+            // Clear $link fields for first pass
             for (const key in docForInsert) {
                 if (isPlainObject(docForInsert[key]) && docForInsert[key].$link) {
                     docForInsert[key] = null;
@@ -4368,11 +4441,18 @@ export async function installPack(packId, user, lang) {
             delete docForInsert._id;
             delete docForInsert._temp_pack_id;
 
-            docForInsert._user = user.username;
+            if (user) docForInsert._user = username;
             docForInsert._model = modelName;
             docForInsert._hash = getFieldValueHash(modelDefForHash, docForInsert);
 
-            const existingDoc = await collection.findOne({ _hash: docForInsert._hash, _user: user.username, _model: modelName }, { projection: { _id: 1 } });
+            // Check for existing document
+            const existingQuery = {
+                _hash: docForInsert._hash,
+                _model: modelName
+            };
+            if (user) existingQuery._user = username;
+
+            const existingDoc = await collection.findOne(existingQuery, { projection: { _id: 1 } });
             if (existingDoc) {
                 tempIdToNewIdMap[tempId] = existingDoc._id;
                 summary.datas.skipped++;
@@ -4398,7 +4478,6 @@ export async function installPack(packId, user, lang) {
                         tempIdToNewIdMap[doc._temp_pack_id_for_mapping] = result.insertedIds[index];
                     }
                 });
-
             } catch (e) {
                 summary.datas.failed += docsToInsert.length;
                 errors.push(`Error inserting batch for ${modelName}: ${e.message}`);
@@ -4407,83 +4486,79 @@ export async function installPack(packId, user, lang) {
         }
     }
 
-    // --- PASSE 2: LIAISON DES RÉFÉRENCES ---
-    logger.info(`[Pack Install] Starting Pass 2: Linking ${linkQueue.length} references.`);
+    // --- PASS 2: REFERENCE LINKING ---
+    logger.info(`[Pack Install] Starting Pass 2: Linking ${linkQueue.length} references`);
     for (const linkOp of linkQueue) {
-        // CORRECTION 2: On récupère le nom du modèle source
         const { sourceTempId, sourceModelName, fieldName, linkSelector } = linkOp;
-
         const sourceId = tempIdToNewIdMap[sourceTempId];
+
         if (!sourceId) {
             logger.warn(`[LINK FAILED] Could not find newly inserted document for temp ID ${sourceTempId}. Skipping link.`);
             continue;
         }
 
-        const cacheKey = JSON.stringify({ selector: linkSelector, user: user.username });
-        let targetIds = null; // Renommé en 'targetIds' car c'est toujours un tableau
-
-        const targetModelName = linkSelector._model;
-        delete linkSelector['_model'];
+        try {
+            const targetModelName = linkSelector._model;
+            delete linkSelector._model;
 
-        // CORRECTION 3: On récupère la définition du modèle SOURCE
-        const sourceModelDef = await getModel(sourceModelName, user);
+            const sourceModelDef = await getModel(sourceModelName, user);
+            const fieldDef = sourceModelDef.fields.find(f => f.name === fieldName);
 
-        try {
-            if (linkCache.has(cacheKey)) {
-                targetIds = linkCache.get(cacheKey);
-                logger.debug(`[LINK CACHE HIT] for ${cacheKey}`);
-            } else {
-                const finalSelector = { ...linkSelector };
-                delete finalSelector['_model']; // nécessaire
-                // CORRECTION 4: Appel corrigé à searchData
-                const { data: targetDocs } = await searchData({model: targetModelName, filter: finalSelector }, user);
-
-                if (targetDocs && targetDocs.length > 0) {
-                    targetIds = targetDocs.map(d => d._id); // Récupère un tableau d'ObjectIds
-                    linkCache.set(cacheKey, targetIds);
-                }
+            if (!fieldDef) {
+                logger.warn(`[LINK FAILED] Field '${fieldName}' not found in source model '${sourceModelName}'`);
+                errors.push(`[LINK FAILED] Field '${fieldName}' not found in source model '${sourceModelName}'`);
+                summary.datas.failed++;
+                continue;
             }
 
-            if (targetIds && targetIds.length > 0) {
-                // CORRECTION 5: On cherche le champ dans la définition du modèle SOURCE
-                const fieldDef = sourceModelDef.fields.find(f => f.name === fieldName);
-                if (!fieldDef) {
-                    logger.warn(`[LINK FAILED] Field '${fieldName}' not found in source model '${sourceModelName}' for doc ${sourceId}.`);
-                    errors.push(`[LINK FAILED] Field '${fieldName}' not found in source model '${sourceModelName}' for doc ${sourceId}.`);
-                    summary.datas.failed++;
-                    continue;
-                }
-
-                // CORRECTION 6: On gère correctement les cas multiples et uniques
-                const valueToSet = fieldDef.multiple ? targetIds.map(id => id.toString()) : targetIds[0].toString();
+            // Search for target documents
+            const { data: targetDocs } = await searchData(
+                { model: targetModelName, filter: linkSelector },
+                user
+            );
 
-                await collection.updateOne(
-                    { _id: sourceId },
-                    { $set: { [fieldName]: valueToSet } }
-                );
-                summary.datas.updated++;
-            } else {
-                const errorMsg = `[LINK FAILED] Could not find target document for linking: ${JSON.stringify(linkSelector)}`;
+            if (!targetDocs || targetDocs.length === 0) {
+                const errorMsg = `[LINK FAILED] No target found for ${JSON.stringify(linkSelector)}`;
                 logger.warn(errorMsg);
                 errors.push(errorMsg);
                 summary.datas.failed++;
+                continue;
             }
+
+            // Update source document with reference
+            const valueToSet = fieldDef.multiple
+                ? targetDocs.map(d => d._id.toString())
+                : targetDocs[0]._id.toString();
+
+            await collection.updateOne(
+                { _id: sourceId },
+                { $set: { [fieldName]: valueToSet } }
+            );
+            summary.datas.updated++;
+
         } catch (e) {
-            const errorMsg = `[LINK CRITICAL] Error during linking for doc ${sourceId}: ${e.message}`;
+            const errorMsg = `[LINK CRITICAL] Error linking ${sourceModelName}.${fieldName}: ${e.message}`;
             logger.error(errorMsg, e.stack);
             errors.push(errorMsg);
             summary.datas.failed++;
         }
     }
 
-    Event.Trigger("OnPackInstalled", "event", "system", pack);
+    // Trigger event only if pack came from database (original behavior)
+    if (typeof packIdentifier === 'string') {
+        await Event.Trigger("OnPackInstalled", "event", "system", pack);
+    }
 
     const modifiedCount = summary.datas.inserted + summary.datas.updated;
-    logger.info(`--- Installation of pack '${pack.name}' finished. ---`);
-    return { success: errors.length === 0, summary, errors, modifiedCount };
+    logger.info(`--- ${logPrefix} completed ---`);
+    return {
+        success: errors.length === 0,
+        summary,
+        errors,
+        modifiedCount
+    };
 }
 
-
 export const installAllPacks = async () => {
     const packs = await getAllPacks();
     await packsCollection.deleteMany({ _user: { $exists : false }});
@@ -4528,14 +4603,7 @@ export async function handleDemoInitialization(req, res) {
         logger.info(`[Demo Init] Installing dynamically generated pack with models: [${models.join(', ')}].`);
 
         // Create and install pack
-        const packsCollection = getCollection('packs');
-        const tempPack = { ...packToInstall, _user: 'system_temp' };
-        const tempInsert = await packsCollection.insertOne(tempPack);
-        const packId = tempInsert.insertedId;
-
-        const result = await installPack(packId, user, req.query.lang || 'en');
-
-        await packsCollection.deleteOne({ _id: packId });
+        const result = await installPack(packToInstall, user, req.query.lang || 'en');
 
         if (result.success || result.modifiedCount > 0) {
             logger.info(`[Demo Init] Pack installed successfully for user '${user.username}'.`);