data-primals-engine 1.0.8 → 1.0.9

package/.github/workflows/node.js.yml

@@ -0,0 +1,59 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "develop", "main" ]
+  pull_request:
+    branches: [ "develop", "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    container:
+      image: node:18  # Specify the Docker version you needx
+      env:
+        NODE_OPTIONS: "--max_old_space_size=4096"
+        NODE_ENV: development
+      ports:
+        - 7635
+        - 27017
+    strategy:
+      matrix:
+        node-version: [18.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+    services:
+      mongodb:
+        image: mongo:5.0
+        ports:
+          - 27017:27017
+        options: >-
+          --health-cmd "mongo --eval 'db.serverStatus()'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install MongoDB Tools
+      run: |
+        wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add -
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list
+        apt-get update
+        apt-get install -y mongodb-org-tools
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '18.18.2'
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Run tests
+      run: npm test
+
+      

package/.github/workflows/npm-publish.yml

@@ -0,0 +1,23 @@
+name: Publish package to GitHub Packages
+on:
+  release:
+    types: [published]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      # Setup .npmrc file to publish to GitHub Packages
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+          registry-url: 'https://npm.pkg.github.com'
+          # Defaults to the user or organization that owns the workflow file
+          scope: '@octocat'
+      - run: npm ci
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/Dockerfile

@@ -0,0 +1,15 @@
+# Base image
+FROM node:18.18.2
+
+# Set working directory
+WORKDIR /
+
+# Copy package files and install dependencies
+COPY package.json ./
+RUN npm install
+
+# Copy application code
+COPY . .
+
+# Expose the application port
+EXPOSE 7633

package/DockerfileTest

@@ -0,0 +1,15 @@
+# Base image
+FROM node:18.18.2
+
+# Set working directory
+WORKDIR /
+
+# Copy package files and install dependencies
+COPY package.json ./
+RUN npm install
+
+# Copy application code
+COPY . .
+
+# Expose the application port
+EXPOSE 7635

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "data-primals-engine",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "data-primals-engine is the package responsible from handling large amount of data in a practical and performant way. It can handle large amount of data in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation.",
   "main": "src/engine.js",
   "type": "module",
   "scripts": {
+    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
     "preinstall": "npx force-resolutions",
     "devserver": "cross-env NODE_ENV=development node server.js",
     "server": "cross-env NODE_ENV=production node server.js",
@@ -39,11 +40,9 @@
     "chalk": "^5.4.1",
     "check-disk-space": "^3.4.0",
     "compression": "^1.8.0",
-    "connect-mongo": "^5.1.0",
     "cookie-parser": "^1.4.7",
     "cronstrue": "^3.1.0",
     "csv-parse": "^5.6.0",
-    "csv-parser": "^3.2.0",
     "date-fns": "^4.1.0",
     "express": "^4.21.2",
     "express-csrf-double-submit-cookie": "^1.2.1",
@@ -61,7 +60,6 @@
     "nodemailer": "^6.10.0",
     "openai": "^5.8.2",
     "randomcolor": "^0.6.2",
-    "rate-limiter-flexible": "^5.0.5",
     "react-helmet": "^6.1.0",
     "react-i18next": "^15.4.1",
     "react-markdown": "^10.1.0",
@@ -70,10 +68,9 @@
     "sirv": "^3.0.1",
     "swagger-ui-express": "^5.0.1",
     "tar": "^7.4.3",
-    "tar-fs": "^3.0.8",
     "uniqid": "^5.4.0",
-    "yaml": "^2.8.0",
-    "zlib": "^1.0.5"
+    "vitest": "^3.2.3",
+    "yaml": "^2.8.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.17.0",

package/src/engine.js

@@ -17,7 +17,7 @@ import formidableMiddleware from 'express-formidable';
 const isProduction = process.env.NODE_ENV === 'production'
 
 // Connection URL
-export const dbUrl = process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017';
+export const dbUrl = process.env.CI ? 'mongodb://mongodb:27017' : (process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017');
 export const MongoClient = new InternalMongoClient(dbUrl, { maxPoolSize: 20 });
 
 // Database Name
@@ -87,7 +87,7 @@ export const Engine = {
             }
         };
 
-        await Promise.all(Config.Get('modules').map(async module => {
+        await Promise.all(Config.Get('modules', []).map(async module => {
             try {
                 if( fs.existsSync(module)){
                     return await importModule(module);

package/src/migrate.js

@@ -15,7 +15,7 @@ Config.Set("modules", ["mongodb"]);
 const MIGRATIONS_DIR = path.resolve(process.cwd(), 'server', 'src', 'migrations');
 const MIGRATIONS_COLLECTION = 'migrations_log';
 
-const engine = Engine.Create();
+const engine = await Engine.Create();
 const port = process.env.MIGRATE_PORT || 7640;
 
 engine.start(port, async () => {

package/src/modules/data.js

@@ -101,7 +101,7 @@ const sseConnections = new Map();
 
 const delay = ms => new Promise(res => setTimeout(res, ms));
 
-const backupDir = process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
+const getBackupDir = () => process.env.BACKUP_DIR || './backups'; // Répertoire de stockage des sauvegardes
 const execAsync = promisify(exec);
 
 let importJobs = {};
@@ -3003,10 +3003,14 @@ async function processRelations(docToProcess, model, collection, me, idMap) {
     // Phase 3: Traitement des résultats
     findResults.forEach((result, index) => {
         const { field, multiple } = batchFinds[index];
-        if (result.data?.length) {
+        if (result.data?.length > 0) {
+            // Cas où des documents sont trouvés
             docToProcess[field] = multiple
                 ? result.data.map(r => r._id.toString())
                 : result.data[0]._id.toString();
+        } else {
+            // Cas où AUCUN document n'est trouvé : il faut nettoyer le champ !
+            docToProcess[field] = multiple ? [] : null;
         }
     });
 
@@ -4940,6 +4944,7 @@ export const loadFromDump = async (user, options = {}) => {
     // ...
     let backupFilePath; // Assurez-vous que cette variable est bien définie avec le chemin du fichier .tar.gz
     // Exemple simplifié :
+    const backupDir = getBackupDir();
     const backupFilenameRegex = new RegExp(`^backup_${user.username}_(\\d+)\\.tar\\.gz$`);
     const backupFiles = fs.readdirSync(backupDir).filter(filename => backupFilenameRegex.test(filename));
     if (backupFiles.length === 0) throw new Error(`Aucun fichier de sauvegarde local trouvé pour l'utilisateur ${user.username}.`);
@@ -5016,6 +5021,7 @@ export const loadFromDump = async (user, options = {}) => {
 
 // Fonction pour générer une clé aléatoire et la stocker dans un fichier
 const generateAndStoreKey = (user) => {
+    const backupDir = getBackupDir();
     const keyFile = path.join(backupDir, getObjectHash({id:getUserId(user)})+'_encryption.key');
     const key = crypto.randomBytes(16).toString('hex');
     fs.writeFileSync(keyFile, key, { mode: 0o600 }); // Permissions strictes
@@ -5024,6 +5030,7 @@ const generateAndStoreKey = (user) => {
 
 // Fonction pour lire la clé depuis le fichier
 const readKeyFromFile = (user) => {
+    const backupDir = getBackupDir();
     const keyFile = path.join(backupDir, getObjectHash({id:getUserId(user)})+'_encryption.key');
     if (fs.existsSync(keyFile)) {
         return fs.readFileSync(keyFile, 'utf8');
@@ -5035,6 +5042,7 @@ export const dumpUserData = async (user) => {
 // Déterminer la clé de chiffrement
     // Pour cet exemple, on simule la config S3. Remplace par la vraie récupération.
     const s3Config = user.configS3; // Supposons que l'objet 'user' passé contient déjà 'configS3'
+    const backupDir = getBackupDir();
 
     let encryptedKey = readKeyFromFile(user);
     if (!encryptedKey) {
@@ -5080,7 +5088,7 @@ export const dumpUserData = async (user) => {
         let col;
         for (const collection of collections) {
 
-            const colls = ['datas', 'models'];
+            const colls = [getUserCollectionName(user), 'models'];
             if( colls.includes(collection.name) ){
 
                 // Exécuter mongodump avec les filtres appropriés
@@ -5166,6 +5174,7 @@ async function manageBackupRotation(user, backupFrequency, s3Config = null) { //
 
     } else {
         logger.info(`Gestion de la rotation des sauvegardes locales pour ${userId}.`);
+        const backupDir = getBackupDir();
         const localFiles = fs.readdirSync(backupDir);
         filesToManage = localFiles
             .filter(f => !fs.lstatSync(path.join(backupDir, f)).isDirectory() && f.startsWith(`backup_${userId}_`) && f.endsWith('.tar.gz'))

package/src/modules/mongodb.js

@@ -23,13 +23,6 @@ export async function onInit(defaultEngine) {
     } catch (e) {
 
     }
-    // Create a SecureContext object
-    // Connection URL
-    const dbUrl = process.env.MONGO_DB_URL || 'mongodb://localhost:27017';
-    const MongoClient = new InternalMongoClient(dbUrl, {
-        tls: false, maxPoolSize: 20
-    });
-    await MongoClient.connect();
 
     modelsCollection = MongoDatabase.collection("models");
     datasCollection = MongoDatabase.collection("datas");

package/src/setenv.js

@@ -0,0 +1,39 @@
+import {getRandom} from "data-primals-engine/core";
+import process from "node:process";
+import {Engine} from "./index.js";
+
+let ports = [], engineInstance, mongod;
+export const getUniquePort = () =>{
+    let d, it=0;
+    do{
+        d = getRandom(10000, 20000);
+        ++it;
+    } while( ports.includes(d) && it < 10000);
+    return d;
+}
+
+// --- Utilitaires pour les tests ---
+export const generateUniqueName = (baseName) => `${baseName}_${getRandom(1000, 9999)}_${Date.now()}`;
+export const initEngine = async () => {
+    if( engineInstance )
+        return engineInstance;
+
+    process.env.OPENAI_API_KEY = "O000";
+
+    const port = process.env.PORT || getUniquePort(); // Different port for this test suite
+    engineInstance = await Engine.Create();
+    await engineInstance.start(port);
+    return engineInstance;
+}
+
+
+/**
+ * Stops the application engine and the in-memory MongoDB instance.
+ */
+export const stopEngine = async () => {
+    if (engineInstance) {
+        await engineInstance.stop();
+        engineInstance = null;
+        console.log("Test engine stopped.");
+    }
+};

package/test/data.backup.integration.test.js

@@ -0,0 +1,141 @@
+// test/data.backup.integration.test.js
+
+import path from "node:path";
+import { Config } from '../src/config.js';
+
+import { ObjectId } from 'mongodb';
+import {expect, describe, it, beforeAll, afterAll, beforeEach} from 'vitest';
+import { vi } from 'vitest'
+import { Buffer } from 'node:buffer'; // Explicitly import Buffer
+import crypto from 'node:crypto';  //Explicitly import crypto
+
+import {
+    createModel,
+    getModel, insertData
+} from 'data-primals-engine/modules/data';
+
+import {
+    modelsCollection as getAppModelsCollection,
+    getCollectionForUser as getAppUserCollection,
+} from 'data-primals-engine/modules/mongodb';
+import { Engine } from "data-primals-engine/engine";
+import process from "node:process";
+
+import { dumpUserData, loadFromDump, getUserHash } from 'data-primals-engine/modules/data';
+import fs from "node:fs";
+import {getRandom} from "data-primals-engine/core";
+import {getUniquePort, initEngine, stopEngine} from "../src/setenv.js";
+
+vi.mock('data-primals-engine/engine', async(importOriginal) => {
+    const mod = await importOriginal() // type is inferred
+    return {
+        ...mod
+    };
+});
+
+// Mock data and settings
+const mockUser = {
+    username: 'testuserBackup',
+    _user: 'testuserBackup',
+    userPlan: 'premium',
+    email: 'testBackup@example.com',
+    configS3: {
+        bucketName: null
+    }
+};
+const testDbName = 'testIntegrationDbHO_Backup';
+const testModelDefinition = {
+    name: 'backupTestModel',
+    _user: mockUser.username,
+    description: 'Model for testing backup/restore',
+    fields: [
+        { name: 'testField', type: 'string', required: true },
+        { name: 'optionalField', type: 'number' },
+    ],
+    maxRequestData: 10,
+};
+
+let testModelsColInstance;
+let testDatasColInstance;
+let engineInstance;
+let testDatasApi;
+
+const backupDir = path.resolve('./test-backups'); // Use an absolute path
+
+beforeAll(async () => {
+
+    process.env.BACKUP_DIR = backupDir; // Set backup directory
+
+    // Create the backup directory if it doesn't exist
+    if (!fs.existsSync(backupDir)) {
+        fs.mkdirSync(backupDir, { recursive: true });
+    }
+
+    // Delete any existing files in the backup directory
+    fs.readdirSync(backupDir).forEach(file => {
+        fs.unlinkSync(path.join(backupDir, file));
+    });
+    vi.stubEnv('S3_CONFIG_ENCRYPTION_KEY', '00000000000000000000000000000000');
+    vi.stubEnv('OPENAI_API_KEY', '00000000000000000000000000000000');
+    // You might need to create a model first if your dumpUserData requires it
+    await createModel(testModelDefinition);
+}, 45000);
+
+afterAll(async () => {
+
+    delete process.env.DB_URL;
+    delete process.env.DB_NAME;
+
+    // Clean up test backups
+    if (fs.existsSync(backupDir)) {
+        fs.readdirSync(backupDir).forEach(file => {
+            fs.unlinkSync(path.join(backupDir, file));
+        });
+        // Optional: fs.rmdirSync(backupDir); // Remove the directory itself
+    }
+});
+
+beforeAll(async () =>{
+    Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow","user", "assistant"]);
+    await initEngine();
+})
+beforeEach(async () => {
+    testModelsColInstance = getAppModelsCollection;
+    testDatasColInstance = getAppUserCollection(mockUser);
+});
+
+describe('Data Backup and Restore Integration', () => {
+    it('should dump and restore user data successfully, and verify data integrity', async () => { // Le nom du test est plus précis
+        // 1. Insérer des données à sauvegarder
+        const initialData = { testField: 'Initial Value', optionalField: 123 };
+        const insertResult = await insertData(testModelDefinition.name, initialData, {}, mockUser, false);
+        expect(insertResult.success).toBe(true);
+        const insertedId = insertResult.insertedIds[0];
+
+        // Vérifier que les données existent avant la sauvegarde
+        let docBeforeBackup = await testDatasColInstance.findOne({ _id: new ObjectId(insertedId) });
+        expect(docBeforeBackup).not.toBeNull();
+        expect(docBeforeBackup.testField).toBe('Initial Value');
+
+        // 2. Sauvegarder les données
+        await dumpUserData(mockUser);
+
+        // 3. Simuler une suppression totale des données
+        await testDatasColInstance.deleteMany({ _user: mockUser.username });
+        let docAfterDelete = await testDatasColInstance.findOne({ _id: new ObjectId(insertedId) });
+        expect(docAfterDelete).toBeNull();
+
+        // 4. Restaurer les données
+        await loadFromDump(mockUser);
+
+        // 5. **VÉRIFICATION FINALE** : S'assurer que les données sont restaurées correctement
+        const countAfterRestore = await testDatasColInstance.countDocuments({ _user: mockUser.username });
+        expect(countAfterRestore).toBeGreaterThan(0);
+
+        const docAfterRestore = await testDatasColInstance.findOne({ _user: mockUser.username, _model: testModelDefinition.name });
+        expect(docAfterRestore).not.toBeNull();
+        expect(docAfterRestore.testField).toBe('Initial Value');
+        expect(docAfterRestore.optionalField).toBe(123);
+
+    }, 15000); // Timeout augmenté pour les opérations de fichiers
+});