data-primals-engine 1.0.8 → 1.0.10

package/src/index.js

@@ -10,6 +10,6 @@ export { UserProvider } from './providers.js';
 
 // --- Database & Data Modules ---
 export { datasCollection, filesCollection, modelsCollection, packsCollection } from './modules/mongodb.js';
-export { searchData, insertData, editData, loadFromDump, dumpUserData, validateRestoreRequest, patchData, deleteData, createModel, getModel, getModels } from './modules/data.js';
+export { searchData, insertData, editData, exportData, importData, scheduleAlerts, cancelAlerts, validateModelStructure, installPack, jobDumpUserData, loadFromDump, dumpUserData, validateRestoreRequest, patchData, deleteData, createModel, editModel, deleteModels, getModel, getModels } from './modules/data.js';
 
 

package/src/migrate.js

@@ -15,7 +15,7 @@ Config.Set("modules", ["mongodb"]);
 const MIGRATIONS_DIR = path.resolve(process.cwd(), 'server', 'src', 'migrations');
 const MIGRATIONS_COLLECTION = 'migrations_log';
 
-const engine = Engine.Create();
+const engine = await Engine.Create();
 const port = process.env.MIGRATE_PORT || 7640;
 
 engine.start(port, async () => {

package/src/modules/data.js

@@ -101,7 +101,7 @@ const sseConnections = new Map();
 
 const delay = ms => new Promise(res => setTimeout(res, ms));
 
-const backupDir = process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
+const getBackupDir = () => process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
 const execAsync = promisify(exec);
 
 let importJobs = {};
@@ -1113,6 +1113,111 @@ function applyCronMask(cronString, mask, defaults) {
     return newParts.join(' ');
 }
 
+
+export const editModel = async (user, id, data) => {
+
+    if( !(isDemoUser(user) && Config.Get("useDemoAccounts")) && isLocalUser(user) && !await hasPermission(["API_ADMIN", "API_EDIT_MODEL"], user)){
+        return ({success: false, error: i18n.t('api.permission.editModel', 'Cannot edit models from the API')})
+    }
+
+    const dataModel = data;
+    try {
+        const collection = getCollectionForUser(user);
+        validateModelStructure(dataModel);
+
+        const el = await modelsCollection.findOne({ $and: [
+                {_user: {$exists: true}},
+                { _id: new ObjectId(id) },
+                {$and: [{_user: {$exists: true}}, {$or: [{_user: user._user}, {_user: user.username}]}]
+                }
+            ]});
+
+        if( !el ){
+            return ({success: false, statusCode: 404, error: i18n.t("api.model.notFound", { model: dataModel.name })});
+        }
+
+        // renommage du modèle
+        if (typeof (data.name)==='string'&&el.name !== data.name && data.name ){
+            await collection.updateMany({ _model: el.name }, { $set: { _model: data.name }});
+            await modelsCollection.updateMany({ 'fields' : {
+                    '$elemMatch' : { relation: el.name }
+                }}, {
+                $set : {
+                    'fields.$.relation' : data.name
+                }
+            })
+        }
+
+        const coll = getCollectionForUser(user);
+        // Update indexes
+// Update indexes
+        if (user.userPlan === 'premium') {
+            let indexes = [];
+            try {
+                // On essaie de récupérer les index existants
+                indexes = await coll.indexes();
+            } catch (e) {
+                // Si la collection n'existe pas, c'est normal.
+                // createIndex la créera. Il n'y a juste pas d'index à supprimer.
+                if (e.codeName !== 'NamespaceNotFound') {
+                    throw e; // On relance les autres erreurs
+                }
+            }
+
+            // Le reste de votre logique de gestion d'index peut maintenant s'exécuter en toute sécurité
+            for (const field of data.fields) {
+                const elField = el.fields.find(f => f.name === field.name);
+                if (!elField) continue;
+
+                const index = indexes.find(i => i.key[field.name] === 1 &&
+                    i.partialFilterExpression?._model === el.name &&
+                    i.partialFilterExpression?._user === user.username);
+
+                if (elField.index !== field.index && !field.index) {
+                    if (index) {
+                        await coll.dropIndex(index.name);
+                    }
+                } else if (elField.index !== field.index && field.index) {
+                    if (!index) {
+                        await coll.createIndex({ [field.name]: 1 }, {
+                            partialFilterExpression: {
+                                _model: data.name,
+                                _user: user.username
+                            }
+                        });
+                    }
+                }
+            }
+        }
+        // suppression des données à la suppression des champs
+        const unset = {};
+        el.fields.filter(f=> !dataModel.fields.some(dt => dt.name === f.name)).map(f => f.name).forEach(f => {
+            unset[f] = 1;
+        });
+        await collection.updateMany({ _model: el.name }, { $unset: unset });
+
+        // sauvegarde du modele
+        const set = {...data};
+        delete set['_id'];
+
+        const oid = new ObjectId(id);
+        await modelsCollection.updateOne({_id: oid}, {$set: set});
+
+        modelsCache.del(user.username+'@@'+el.name);
+
+        const model = await modelsCollection.findOne({_id: oid });
+        triggerWorkflows(model, user, 'ModelEdited').catch(workflowError => {
+            logger.error(`Erreur asynchrone lors du déclenchement des workflows pour ${model._model} ID ${model._id}:`, workflowError);
+        });
+
+        return ({ success: true, data: await modelsCollection.findOne({_id : oid}) });
+    } catch (e) {
+        logger.error(e);
+        return ({ success: false, error: e.message, statusCode: 500 });
+    }
+};
+
+
 export async function onInit(defaultEngine) {
     engine = defaultEngine;
     logger = engine.getComponent(Logger);
@@ -1459,95 +1564,17 @@ export async function onInit(defaultEngine) {
     });
 
     engine.put('/api/model/:id', [middlewareAuthenticator, userInitiator, setTimeoutMiddleware(15000)], async (req, res) => {
-
-        if( !(isDemoUser(req.me) && Config.Get("useDemoAccounts")) && isLocalUser(req.me) && !await hasPermission(["API_ADMIN", "API_EDIT_MODEL"], req.me)){
-            return res.status(403).json({success: false, error: i18n.t('api.permission.editModel', 'Cannot edit models from the API')})
-        }
-
-        const dataModel = req.fields;
-        try {
-            const collection = getCollectionForUser(req.me);
-            validateModelStructure(dataModel);
-
-            const el = await modelsCollection.findOne({ $and: [
-                {_user: {$exists: true}},
-                { _id: new ObjectId(req.params.id) },
-                {$and: [{_user: {$exists: true}}, {$or: [{_user: req.me._user}, {_user: req.me.username}]}]
-            }
-            ]});
-
-            if( !el ){
-                return res.status(404).json({error: i18n.t("api.model.notFound", { model: dataModel.name })});
-            }
-
-            // renommage du modèle
-            if (typeof (req.fields.name)==='string'&&el.name !== req.fields.name && req.fields.name ){
-                await collection.updateMany({ _model: el.name }, { $set: { _model: req.fields.name }});
-                await modelsCollection.updateMany({ 'fields' : {
-                    '$elemMatch' : { relation: el.name }
-                    }}, {
-                    $set : {
-                        'fields.$.relation' : req.fields.name
-                    }
-                })
-            }
-
-            // Update indexes
-            if( req.me.userPlan === 'premium') {
-                const indexes = await datasCollection.indexes();
-                for (const field of dataModel.fields) {
-                    const elField = el.fields.find(f => f.name === field.name);
-                    if (!elField)
-                        continue;
-                    const index = indexes.find(i => i.key[field.name] === 1 &&
-                        i.partialFilterExpression?._model === el.name &&
-                        i.partialFilterExpression?._user === req.me.username);
-                    if (elField.index !== field.index && !field.index) {
-                        if (index) {
-                            await datasCollection.dropIndex(index.name);
-                        }
-                    } else if (elField.index !== field.index && field.index) {
-                        if (!index) {
-                            await datasCollection.createIndex({[field.name]: 1}, {
-                                partialFilterExpression: {
-                                    _model: dataModel.name,
-                                    _user: req.me.username
-                                }
-                            });
-                        }
-                    }
-                }
-            }
-
-            // suppression des données à la suppression des champs
-            const unset = {};
-            el.fields.filter(f=> !dataModel.fields.some(dt => dt.name === f.name)).map(f => f.name).forEach(f => {
-                unset[f] = 1;
-            });
-            await collection.updateMany({ _model: el.name }, { $unset: unset });
-
-            // sauvegarde du modele
-            const set = {...req.fields};
-            delete set['_id'];
-            await modelsCollection.updateOne({_id: new ObjectId(req.params.id)}, {$set: set});
-
-            modelsCache.del(req.me.username+'@@'+el.name);
-
-            const model = await modelsCollection.findOne({_id: new ObjectId(req.params.id) });
-            triggerWorkflows(model, req.me, 'ModelEdited').catch(workflowError => {
-                logger.error(`Erreur asynchrone lors du déclenchement des workflows pour ${model._model} ID ${model._id}:`, workflowError);
-            });
-
-            res.json({ success: true, data: await modelsCollection.findOne({_id : new ObjectId(req.params.id)}) });
-        } catch (e) {
-            logger.error(e);
-            res.json({ success: false });
+        const result = await editModel(req.me, req.params.id, req.fields);
+        if( result.success){
+            return res.status(result.statusCode || '200').json(result);
+        }else{
+            return res.status(result.statusCode || '500').json(result);
         }
-    })
+    });
 
     engine.post('/api/data/restore', [throttle, middlewareAuthenticator, userInitiator,myFreePremiumAnonymousLimiter, setTimeoutMiddleware(60000)], async (req, res) => {
 
-        if (!((req.me?.roles || []).includes("admin"))) {
+        if (!((user?.roles || []).includes("admin"))) {
             return res.status(403).json({success: false, error: 'Cannot backup data. Contact an administrator to get back your data'})
         }
 
@@ -3003,10 +3030,14 @@ async function processRelations(docToProcess, model, collection, me, idMap) {
     // Phase 3: Traitement des résultats
     findResults.forEach((result, index) => {
         const { field, multiple } = batchFinds[index];
-        if (result.data?.length) {
+        if (result.data?.length > 0) {
+            // Cas où des documents sont trouvés
             docToProcess[field] = multiple
                 ? result.data.map(r => r._id.toString())
                 : result.data[0]._id.toString();
+        } else {
+            // Cas où AUCUN document n'est trouvé : il faut nettoyer le champ !
+            docToProcess[field] = multiple ? [] : null;
         }
     });
 
@@ -4940,6 +4971,7 @@ export const loadFromDump = async (user, options = {}) => {
     // ...
     let backupFilePath; // Assurez-vous que cette variable est bien définie avec le chemin du fichier .tar.gz
     // Exemple simplifié :
+    const backupDir = getBackupDir();
     const backupFilenameRegex = new RegExp(`^backup_${user.username}_(\\d+)\\.tar\\.gz$`);
     const backupFiles = fs.readdirSync(backupDir).filter(filename => backupFilenameRegex.test(filename));
     if (backupFiles.length === 0) throw new Error(`Aucun fichier de sauvegarde local trouvé pour l'utilisateur ${user.username}.`);
@@ -5016,6 +5048,7 @@ export const loadFromDump = async (user, options = {}) => {
 
 // Fonction pour générer une clé aléatoire et la stocker dans un fichier
 const generateAndStoreKey = (user) => {
+    const backupDir = getBackupDir();
     const keyFile = path.join(backupDir, getObjectHash({id:getUserId(user)})+'_encryption.key');
     const key = crypto.randomBytes(16).toString('hex');
     fs.writeFileSync(keyFile, key, { mode: 0o600 }); // Permissions strictes
@@ -5024,6 +5057,7 @@ const generateAndStoreKey = (user) => {
 
 // Fonction pour lire la clé depuis le fichier
 const readKeyFromFile = (user) => {
+    const backupDir = getBackupDir();
     const keyFile = path.join(backupDir, getObjectHash({id:getUserId(user)})+'_encryption.key');
     if (fs.existsSync(keyFile)) {
         return fs.readFileSync(keyFile, 'utf8');
@@ -5035,6 +5069,7 @@ export const dumpUserData = async (user) => {
 // Déterminer la clé de chiffrement
     // Pour cet exemple, on simule la config S3. Remplace par la vraie récupération.
     const s3Config = user.configS3; // Supposons que l'objet 'user' passé contient déjà 'configS3'
+    const backupDir = getBackupDir();
 
     let encryptedKey = readKeyFromFile(user);
     if (!encryptedKey) {
@@ -5080,7 +5115,7 @@ export const dumpUserData = async (user) => {
         let col;
         for (const collection of collections) {
 
-            const colls = ['datas', 'models'];
+            const colls = [getUserCollectionName(user), 'models'];
             if( colls.includes(collection.name) ){
 
                 // Exécuter mongodump avec les filtres appropriés
@@ -5166,6 +5201,7 @@ async function manageBackupRotation(user, backupFrequency, s3Config = null) { //
 
     } else {
         logger.info(`Gestion de la rotation des sauvegardes locales pour ${userId}.`);
+        const backupDir = getBackupDir();
         const localFiles = fs.readdirSync(backupDir);
         filesToManage = localFiles
             .filter(f => !fs.lstatSync(path.join(backupDir, f)).isDirectory() && f.startsWith(`backup_${userId}_`) && f.endsWith('.tar.gz'))

package/src/modules/mongodb.js

@@ -23,13 +23,6 @@ export async function onInit(defaultEngine) {
     } catch (e) {
 
     }
-    // Create a SecureContext object
-    // Connection URL
-    const dbUrl = process.env.MONGO_DB_URL || 'mongodb://localhost:27017';
-    const MongoClient = new InternalMongoClient(dbUrl, {
-        tls: false, maxPoolSize: 20
-    });
-    await MongoClient.connect();
 
     modelsCollection = MongoDatabase.collection("models");
     datasCollection = MongoDatabase.collection("datas");

package/src/setenv.js

@@ -0,0 +1,39 @@
+import {getRandom} from "data-primals-engine/core";
+import process from "node:process";
+import {Engine} from "./index.js";
+
+let ports = [], engineInstance, mongod;
+export const getUniquePort = () =>{
+    let d, it=0;
+    do{
+        d = getRandom(10000, 20000);
+        ++it;
+    } while( ports.includes(d) && it < 10000);
+    return d;
+}
+
+// --- Utilitaires pour les tests ---
+export const generateUniqueName = (baseName) => `${baseName}_${getRandom(1000, 9999)}_${Date.now()}`;
+export const initEngine = async () => {
+    if( engineInstance )
+        return engineInstance;
+
+    process.env.OPENAI_API_KEY = "O000";
+
+    const port = process.env.PORT || getUniquePort(); // Different port for this test suite
+    engineInstance = await Engine.Create();
+    await engineInstance.start(port);
+    return engineInstance;
+}
+
+
+/**
+ * Stops the application engine and the in-memory MongoDB instance.
+ */
+export const stopEngine = async () => {
+    if (engineInstance) {
+        await engineInstance.stop();
+        engineInstance = null;
+        console.log("Test engine stopped.");
+    }
+};

package/test/data.backup.integration.test.js

@@ -0,0 +1,141 @@
+// test/data.backup.integration.test.js
+
+import path from "node:path";
+import { Config } from '../src/config.js';
+
+import { ObjectId } from 'mongodb';
+import {expect, describe, it, beforeAll, afterAll, beforeEach} from 'vitest';
+import { vi } from 'vitest'
+import { Buffer } from 'node:buffer'; // Explicitly import Buffer
+import crypto from 'node:crypto';  //Explicitly import crypto
+
+import {
+    createModel,
+    getModel, insertData
+} from 'data-primals-engine/modules/data';
+
+import {
+    modelsCollection as getAppModelsCollection,
+    getCollectionForUser as getAppUserCollection,
+} from 'data-primals-engine/modules/mongodb';
+import { Engine } from "data-primals-engine/engine";
+import process from "node:process";
+
+import { dumpUserData, loadFromDump, getUserHash } from 'data-primals-engine/modules/data';
+import fs from "node:fs";
+import {getRandom} from "data-primals-engine/core";
+import {getUniquePort, initEngine, stopEngine} from "../src/setenv.js";
+
+vi.mock('data-primals-engine/engine', async(importOriginal) => {
+    const mod = await importOriginal() // type is inferred
+    return {
+        ...mod
+    };
+});
+
+// Mock data and settings
+const mockUser = {
+    username: 'testuserBackup',
+    _user: 'testuserBackup',
+    userPlan: 'premium',
+    email: 'testBackup@example.com',
+    configS3: {
+        bucketName: null
+    }
+};
+const testDbName = 'testIntegrationDbHO_Backup';
+const testModelDefinition = {
+    name: 'backupTestModel',
+    _user: mockUser.username,
+    description: 'Model for testing backup/restore',
+    fields: [
+        { name: 'testField', type: 'string', required: true },
+        { name: 'optionalField', type: 'number' },
+    ],
+    maxRequestData: 10,
+};
+
+let testModelsColInstance;
+let testDatasColInstance;
+let engineInstance;
+let testDatasApi;
+
+const backupDir = path.resolve('./test-backups'); // Use an absolute path
+
+beforeAll(async () => {
+
+    process.env.BACKUP_DIR = backupDir; // Set backup directory
+
+    // Create the backup directory if it doesn't exist
+    if (!fs.existsSync(backupDir)) {
+        fs.mkdirSync(backupDir, { recursive: true });
+    }
+
+    // Delete any existing files in the backup directory
+    fs.readdirSync(backupDir).forEach(file => {
+        fs.unlinkSync(path.join(backupDir, file));
+    });
+    vi.stubEnv('S3_CONFIG_ENCRYPTION_KEY', '00000000000000000000000000000000');
+    vi.stubEnv('OPENAI_API_KEY', '00000000000000000000000000000000');
+    // You might need to create a model first if your dumpUserData requires it
+    await createModel(testModelDefinition);
+}, 45000);
+
+afterAll(async () => {
+
+    delete process.env.DB_URL;
+    delete process.env.DB_NAME;
+
+    // Clean up test backups
+    if (fs.existsSync(backupDir)) {
+        fs.readdirSync(backupDir).forEach(file => {
+            fs.unlinkSync(path.join(backupDir, file));
+        });
+        // Optional: fs.rmdirSync(backupDir); // Remove the directory itself
+    }
+});
+
+beforeAll(async () =>{
+    Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow","user", "assistant"]);
+    await initEngine();
+})
+beforeEach(async () => {
+    testModelsColInstance = getAppModelsCollection;
+    testDatasColInstance = getAppUserCollection(mockUser);
+});
+
+describe('Data Backup and Restore Integration', () => {
+    it('should dump and restore user data successfully, and verify data integrity', async () => { // Le nom du test est plus précis
+        // 1. Insérer des données à sauvegarder
+        const initialData = { testField: 'Initial Value', optionalField: 123 };
+        const insertResult = await insertData(testModelDefinition.name, initialData, {}, mockUser, false);
+        expect(insertResult.success).toBe(true);
+        const insertedId = insertResult.insertedIds[0];
+
+        // Vérifier que les données existent avant la sauvegarde
+        let docBeforeBackup = await testDatasColInstance.findOne({ _id: new ObjectId(insertedId) });
+        expect(docBeforeBackup).not.toBeNull();
+        expect(docBeforeBackup.testField).toBe('Initial Value');
+
+        // 2. Sauvegarder les données
+        await dumpUserData(mockUser);
+
+        // 3. Simuler une suppression totale des données
+        await testDatasColInstance.deleteMany({ _user: mockUser.username });
+        let docAfterDelete = await testDatasColInstance.findOne({ _id: new ObjectId(insertedId) });
+        expect(docAfterDelete).toBeNull();
+
+        // 4. Restaurer les données
+        await loadFromDump(mockUser);
+
+        // 5. **VÉRIFICATION FINALE** : S'assurer que les données sont restaurées correctement
+        const countAfterRestore = await testDatasColInstance.countDocuments({ _user: mockUser.username });
+        expect(countAfterRestore).toBeGreaterThan(0);
+
+        const docAfterRestore = await testDatasColInstance.findOne({ _user: mockUser.username, _model: testModelDefinition.name });
+        expect(docAfterRestore).not.toBeNull();
+        expect(docAfterRestore.testField).toBe('Initial Value');
+        expect(docAfterRestore.optionalField).toBe(123);
+
+    }, 15000); // Timeout augmenté pour les opérations de fichiers
+});