data-primals-engine 1.2.6-rc1 → 1.2.6-rc3

package/README.md

@@ -315,7 +315,7 @@ curl -X DELETE http://localhost:7633/api/data?_user=demo \
 Make sure you use the code below to initialize the user : 
 ```javascript
 import express from "express";
-import {Engine, insertData, searchData } from './src/index';
+import {Engine, insertData, searchData } from 'data-primals-engine';
 
 // Ensure the engine is initialized
 

package/package.json

@@ -1,120 +1,120 @@
-{
-  "name": "data-primals-engine",
-  "version": "1.2.6-rc1",
-  "description": "data-primals-engine is a package responsible from handling large amount of data using MongoDB in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation. It also has integrated AI assistant.",
-  "main": "src/engine.js",
-  "type": "module",
-  "scripts": {
-    "preinstall": "npx force-resolutions",
-    "dev": "concurrently \"npm:client\" \"npm:devserver\"",
-    "prod": "npm run build-server && npm run server",
-    "client": "cd client && npm run dev",
-    "build-server": "vite build client --config client/vite.config.js --outDir dist",
-    "build-client": "cd client && npm run build",
-    "server": "cross-env NODE_ENV=production node server.js",
-    "devserver": "npm run build-server && cross-env NODE_ENV=development PORT=7633 node server.js",
-    "lint": "eslint .",
-    "format": "prettier --write .",
-    "clean": "rm -rf node_modules package-lock.json",
-    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
-    "audit": "npm audit --audit-level=high",
-    "migrate:create": "node src/migrate.js create",
-    "migrate:up": "node src/migrate.js up",
-    "migrate:down": "node src/migrate.js down",
-    "migrate:revert": "node src/migrate.js revert",
-    "migrate:status": "node src/migrate.js status"
-  },
-  "optionalDependencies": {
-    "@rollup/rollup-linux-x64-gnu": "4.6.1"
-  },
-  "resolutions": {
-    "tar-fs": "3.0.9",
-    "on-headers": "1.1.0",
-    "brace-expansion": "2.0.2",
-    "prismjs": "1.30.0"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/anonympins/data-primals-engine.git"
-  },
-  "exports": {
-    ".": "./src/index.js",
-    "./modules/*.js": "./src/modules/*.js",
-    "./modules/*": "./src/modules/*/index.js",
-    "./client": "./client/index.js",
-    "./*": "./src/*.js"
-  },
-  "peerDependencies": {
-    "express": "^5.1.0",
-    "react": ">=18.0.0",
-    "react-query": ">=3.0.0"
-  },
-  "dependencies": {
-    "@langchain/core": "^0.3.66",
-    "@langchain/deepseek": "^0.1.0",
-    "@langchain/google-genai": "^0.2.16",
-    "@langchain/openai": "^0.6.3",
-    "archiver": "^7.0.1",
-    "aws-sdk": "^2.1692.0",
-    "bcrypt": "^6.0.0",
-    "body-parser": "^2.2.0",
-    "chalk": "^5.4.1",
-    "check-disk-space": "^3.4.0",
-    "compression": "^1.8.1",
-    "cookie-parser": "^1.4.7",
-    "cronstrue": "^3.2.0",
-    "csv-parse": "^6.1.0",
-    "date-fns": "^4.1.0",
-    "express-csrf-double-submit-cookie": "^2.0.0",
-    "express-formidable": "^1.2.0",
-    "express-mongo-sanitize": "^2.2.0",
-    "express-rate-limit": "^8.0.1",
-    "express-session": "^1.18.2",
-    "i18next-browser-languagedetector": "^8.2.0",
-    "isolated-vm": "^4.7.2",
-    "juice": "^11.0.1",
-    "mathjs": "^14.6.0",
-    "mongodb": "^6.18.0",
-    "node-cache": "^5.1.2",
-    "node-schedule": "^2.1.1",
-    "nodemailer": "^7.0.5",
-    "openai": "^5.10.2",
-    "process": "^0.11.10",
-    "prop-types": "^15.8.1",
-    "randomcolor": "^0.6.2",
-    "react-i18next": "^15.6.1",
-    "react-markdown": "^10.1.0",
-    "read-excel-file": "^5.8.8",
-    "request-ip": "^3.3.0",
-    "sanitize-html": "^2.17.0",
-    "sirv": "^3.0.1",
-    "swagger-ui-express": "^5.0.1",
-    "tar": "^7.4.3",
-    "uniqid": "^5.4.0",
-    "vitest": "^3.2.4",
-    "yaml": "^2.8.0"
-  },
-  "devDependencies": {
-    "@eslint/js": "^9.32.0",
-    "concurrently": "^9.2.0",
-    "cross-env": "^10.0.0",
-    "eslint": "^9.32.0",
-    "globals": "^16.3.0"
-  },
-  "subPackages": [
-    "client"
-  ],
-  "keywords": [
-    "data-driven engine",
-    "headless CMS",
-    "backend",
-    "automation",
-    "AWS S3",
-    "MongoDB"
-  ],
-  "author": "anonympins",
-  "license": "MIT",
-  "engines": {
-    "node": ">=20.0.0"
-  }
-}
+{
+  "name": "data-primals-engine",
+  "version": "1.2.6-rc3",
+  "description": "data-primals-engine is a package responsible from handling large amount of data using MongoDB in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation. It also has integrated AI assistant.",
+  "main": "src/engine.js",
+  "type": "module",
+  "scripts": {
+    "preinstall": "npx force-resolutions",
+    "dev": "concurrently \"npm:client\" \"npm:devserver\"",
+    "prod": "npm run build-server && npm run server",
+    "client": "cd client && npm run dev",
+    "build-server": "vite build client --config client/vite.config.js --outDir dist",
+    "build-client": "cd client && npm run build",
+    "server": "cross-env NODE_ENV=production node server.js",
+    "devserver": "npm run build-server && cross-env NODE_ENV=development PORT=7633 node server.js",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "clean": "rm -rf node_modules package-lock.json",
+    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
+    "audit": "npm audit --audit-level=high",
+    "migrate:create": "node src/migrate.js create",
+    "migrate:up": "node src/migrate.js up",
+    "migrate:down": "node src/migrate.js down",
+    "migrate:revert": "node src/migrate.js revert",
+    "migrate:status": "node src/migrate.js status"
+  },
+  "optionalDependencies": {
+    "@rollup/rollup-linux-x64-gnu": "4.6.1"
+  },
+  "resolutions": {
+    "tar-fs": "3.0.9",
+    "on-headers": "1.1.0",
+    "brace-expansion": "2.0.2",
+    "prismjs": "1.30.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/anonympins/data-primals-engine.git"
+  },
+  "exports": {
+    ".": "./src/index.js",
+    "./modules/*.js": "./src/modules/*.js",
+    "./modules/*": "./src/modules/*/index.js",
+    "./client": "./client/index.js",
+    "./*": "./src/*.js"
+  },
+  "peerDependencies": {
+    "express": "^5.1.0",
+    "react": ">=18.0.0",
+    "react-query": ">=3.0.0"
+  },
+  "dependencies": {
+    "@langchain/core": "^0.3.66",
+    "@langchain/deepseek": "^0.1.0",
+    "@langchain/google-genai": "^0.2.16",
+    "@langchain/openai": "^0.6.3",
+    "archiver": "^7.0.1",
+    "aws-sdk": "^2.1692.0",
+    "bcrypt": "^6.0.0",
+    "body-parser": "^2.2.0",
+    "chalk": "^5.4.1",
+    "check-disk-space": "^3.4.0",
+    "compression": "^1.8.1",
+    "cookie-parser": "^1.4.7",
+    "cronstrue": "^3.2.0",
+    "csv-parse": "^6.1.0",
+    "date-fns": "^4.1.0",
+    "express-csrf-double-submit-cookie": "^2.0.0",
+    "express-formidable": "^1.2.0",
+    "express-mongo-sanitize": "^2.2.0",
+    "express-rate-limit": "^8.0.1",
+    "express-session": "^1.18.2",
+    "i18next-browser-languagedetector": "^8.2.0",
+    "isolated-vm": "^4.7.2",
+    "juice": "^11.0.1",
+    "mathjs": "^14.6.0",
+    "mongodb": "^6.18.0",
+    "node-cache": "^5.1.2",
+    "node-schedule": "^2.1.1",
+    "nodemailer": "^7.0.5",
+    "openai": "^5.10.2",
+    "process": "^0.11.10",
+    "prop-types": "^15.8.1",
+    "randomcolor": "^0.6.2",
+    "react-i18next": "^15.6.1",
+    "react-markdown": "^10.1.0",
+    "read-excel-file": "^5.8.8",
+    "request-ip": "^3.3.0",
+    "sanitize-html": "^2.17.0",
+    "sirv": "^3.0.1",
+    "swagger-ui-express": "^5.0.1",
+    "tar": "^7.4.3",
+    "uniqid": "^5.4.0",
+    "vitest": "^3.2.4",
+    "yaml": "^2.8.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.32.0",
+    "concurrently": "^9.2.0",
+    "cross-env": "^10.0.0",
+    "eslint": "^9.32.0",
+    "globals": "^16.3.0"
+  },
+  "subPackages": [
+    "client"
+  ],
+  "keywords": [
+    "data-driven engine",
+    "headless CMS",
+    "backend",
+    "automation",
+    "AWS S3",
+    "MongoDB"
+  ],
+  "author": "anonympins",
+  "license": "MIT",
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/server.js

@@ -8,7 +8,7 @@ import {Config, Engine, BenchmarkTool, GameObject, Logger} from "./src/index.js"
 import sirv from "sirv";
 import express from "express";
 
-Config.Set("modules", ["data", "mongodb", "file", "bucket", "workflow","user", "assistant", "swagger"])
+Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow","user", "assistant", "swagger"])
 Config.Set("middlewares", []);
 
 const bench = GameObject.Create("Benchmark");

package/src/engine.js

@@ -1,281 +1,285 @@
-import {GameObject, Logger} from "./gameObject.js";
-import {Config} from "./config.js";
-import fs from 'node:fs'
-import express from 'express'
-import {MongoClient as InternalMongoClient} from 'mongodb'
-import process from "process";
-import {
-    cookiesSecret,
-    databasePoolSize,
-    dbName,
-    tlsAllowInvalidCertificates, tlsAllowInvalidHostnames
-} from "./constants.js";
-import http from "http";
-import cookieParser from "cookie-parser";
-import requestIp from 'request-ip';
-import {createModel, deleteModels, getModels, installAllPacks, validateModelStructure} from "./modules/data/data.js";
-import {defaultModels} from "./defaultModels.js";
-import {DefaultUserProvider} from "./providers.js";
-import formidableMiddleware from 'express-formidable';
-import sirv from "sirv";
-import * as tls from "node:tls";
-import {Event} from "./events.js";
-import path from "node:path";
-import {isPathRelativeTo, isValidPath} from "./core.js";
-
-// Constants
-const isProduction = process.env.NODE_ENV === 'production'
-
-let caFile, certFile, keyFile;
-try {
-    if (process.env.CA_CERT)
-        caFile = fs.readFileSync(process.env.CA_CERT);
-} catch (e) {}
-try {
-    if (process.env.CERT)
-        certFile = fs.readFileSync(process.env.CERT);
-}catch (e) {}
-try{
-    if (process.env.CERT_KEY)
-        keyFile = fs.readFileSync(process.env.CERT_KEY);
-} catch (e) {}
-
-const secureContext = tls.createSecureContext({
-    ca: caFile, cert: certFile, key: keyFile
-});
-
-export const dbUrl = process.env.CI ? 'mongodb://mongodb:27017' : (process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017');
-
-const isTlsActive = !(!process.env.TLS || ["0", "false"].includes(process.env.TLS.toLowerCase()));
-
-const clientOptions = {
-    maxPoolSize: databasePoolSize
-};
-
-// We add TLS options if enabled
-if (isTlsActive) {
-    clientOptions.tls = true;
-
-    // is mTLS ? (client certificate required instead of password)
-    if (process.env.CERT) {
-        clientOptions.secureContext = tls.createSecureContext({
-            ca: fs.readFileSync(process.env.CA_CERT),
-            cert: fs.readFileSync(process.env.CERT),
-            key: fs.readFileSync(process.env.CERT_KEY)
-        });
-    }else {
-        // Path to the authority certificate
-        if (process.env.CA_CERT) {
-            clientOptions.tlsCAFile = process.env.CA_CERT;
-        }
-        // Path to the certificate key
-        if (process.env.CERT_KEY) {
-            clientOptions.tlsCertificateKeyFile = process.env.CERT_KEY;
-        }
-    }
-    if (tlsAllowInvalidCertificates) {
-        clientOptions.tlsAllowInvalidCertificates = true;
-        console.warn("🚨 [SECURITY WARNING] tlsAllowInvalidCertificates is ON. Server certificate will not be validated.");
-    }
-    if (tlsAllowInvalidHostnames) {
-        clientOptions.tlsAllowInvalidHostnames = true;
-        console.warn("🚨 [SECURITY WARNING] tlsAllowInvalidHostnames is ON. Server hostname will not be validated.");
-    }
-}
-
-export const MongoClient = new InternalMongoClient(dbUrl, clientOptions);
-
-
-// Database Name
-export const MongoDatabase = MongoClient.db(dbName);
-
-
-export const Engine = {
-    Create: async (options = { app : null}) => {
-        const engine = GameObject.Create("Engine");
-        console.log("Creating engine", Config.Get('modules'));
-        const logger = engine.addComponent(Logger);
-
-        engine.userProvider = new DefaultUserProvider(engine);
-
-        engine.setUserProvider = (providerInstance) => {
-            engine.userProvider = providerInstance;
-            logger.info(`Custom UserProvider '${providerInstance.constructor.name}' has been set.`);
-        };
-
-        if (!options.app) {
-            options.app = express();
-        }
-
-        const { app }  = options;
-        // Allows you to set port in the project properties.
-        app.set('port', process.env.PORT || 3000);
-        app.set('engine', engine);
-
-        app.use(formidableMiddleware({
-            encoding: 'utf-8',
-            uploadDir: process.cwd()+'/uploads/tmp',
-            multiples: true // req.files to be arrays of files
-        }));
-        app.use(cookieParser(process.env.COOKIES_SECRET || cookiesSecret));
-        app.use(requestIp.mw())
-
-        engine.use = (...args) => {
-            return app.use(...args);
-        }
-        engine.post = (...args) => {
-            return app.post(...args);
-        };
-        engine.get = (...args) => {
-            return app.get(...args);
-        };
-        engine.delete = (...args) => {
-            return app.delete(...args);
-        };
-        engine.patch = (...args) => {
-            return app.patch(...args);
-        };
-        engine.put = (...args) => {
-            return app.put(...args);
-        };
-        engine.all = (...args) => {
-            return app.all(...args);
-        };
-        engine.getModule = (module) => {
-            return engine._modules.find(m => m.module === module);
-        };
-
-        const importModule = async (module) => {
-            const moduleA = await import(module);
-            if (moduleA.onInit){
-                await moduleA.onInit(engine);
-                return {...moduleA, module};
-            }else {
-                const mod = moduleA.default();
-                await mod?.onInit(engine);
-                return { ...mod, module};
-            }
-        };
-
-        await Promise.all(Config.Get('modules', []).map(async moduleIdentifier => {
-            try {
-                let moduleDir;
-                const moduleName = path.basename(moduleIdentifier);
-
-                const directPath = path.resolve(moduleIdentifier);
-                let isDir = fs.existsSync(directPath) && fs.statSync(directPath).isDirectory();
-                if (isDir) {
-                    moduleDir = directPath;
-                    if (!fs.existsSync(moduleDir) || !fs.statSync(moduleDir).isDirectory()) {
-                        logger.warn(`Le dossier du module est introuvable pour l'identifiant : '${moduleIdentifier}'. Chemin cherché : '${moduleDir}'.`);
-                        return null;
-                    }
-                } else {
-                    moduleDir = path.resolve('./src/modules', moduleIdentifier);
-                }
-
-                let moduleEntryPoint;
-                const jsPath = moduleDir+'.js';
-                const indexJsPath = path.join(moduleDir, 'index.js');
-                const moduleJsPath = path.join(moduleDir, `${moduleName}.js`);
-
-                if (fs.existsSync(jsPath)) {
-                    moduleEntryPoint = 'file://'+jsPath;
-                } else if (fs.existsSync(indexJsPath)) {
-                    moduleEntryPoint = 'file://'+indexJsPath;
-                } else if (fs.existsSync(moduleJsPath)) {
-                    moduleEntryPoint = 'file://'+moduleJsPath;
-                }
-
-                return await importModule(moduleEntryPoint);
-            } catch (e) {
-                logger.error(`Échec du chargement du module '${moduleIdentifier}':`, e.stack);
-                return null;
-            }
-        })).then(async results => {
-            // On filtre les modules qui n'ont pas pu être chargés
-            engine._modules = results.filter(Boolean);
-            return Promise.resolve();
-        });
-
-        let server;
-
-        engine.start = async (port, cb) =>{
-            // Use connect method to connect to the server
-
-            // Start http server
-            server = http.createServer(app);
-
-            // Server Timeout Settings
-            server.timeout = 120000;
-            server.headersTimeout = 20000;
-            server.requestTimeout = 30000;
-            server.keepAliveTimeout = 5000;
-
-            server.listen(port);
-
-            await setupInitialModels();
-            await installAllPacks();
-
-            if (cb)
-                await cb();
-
-            engine.get('/api/health', (req, res) => {
-                res.status(200).json({
-                    status: 'ok',
-                    timestamp: new Date().toISOString()
-                });
-            });
-
-            app.use(sirv('client/dist', {
-                single: true,
-                dev: process.env.NODE_ENV === 'development'
-            }));
-
-            process.on('uncaughtException', function (exception) {
-                console.error(exception);
-                fs.appendFile('issues.txt', JSON.stringify({ code: exception.code, message: exception.message, stack: exception.stack }), function (err) {
-                    if (err){
-                        throw err;
-                    }
-                });
-                process.exit(1);
-            });
-
-            Event.Trigger("OnServerStart", "event", "system", engine);
-        }
-
-        engine.stop = async () => {
-            await server.close();
-            Event.Trigger("OnServerStop", "event", "system", engine);
-        };
-
-        async function setupInitialModels() {
-            logger.info("Validating structures of default models...");
-            const ms = Object.values(Config.Get('defaultModels', defaultModels));
-
-            let dbModels = await getModels();
-
-            for(let i = 0; i < ms.length; ++i){
-                const model = ms[i];
-                validateModelStructure(model);
-                // Création des modèles
-                if( !dbModels.find(m =>m.name === model.name) )
-                {
-                    model.locked = true;
-                    const r = await createModel(model);
-                    dbModels.push({...model, _id: r.insertedId });
-                    logger.info('Model inserted (' + model.name + ')');
-                }else
-                    logger.info('Model loaded (' + model.name + ')');
-            }
-            logger.info("All models loaded.");
-            Event.Trigger("OnModelsLoaded", "event", "system", engine, dbModels);
-        }
-        engine.resetModels = async () => {
-            await deleteModels();
-            Event.Trigger("OnModelsDeleted", "event", "system", engine);
-        };
-        return engine;
-    }
-}
+import {GameObject, Logger} from "./gameObject.js";
+import {Config} from "./config.js";
+import fs from 'node:fs'
+import express from 'express'
+import {MongoClient as InternalMongoClient} from 'mongodb'
+import process from "process";
+import {
+    cookiesSecret,
+    databasePoolSize,
+    dbName,
+    tlsAllowInvalidCertificates, tlsAllowInvalidHostnames
+} from "./constants.js";
+import http from "http";
+import cookieParser from "cookie-parser";
+import requestIp from 'request-ip';
+import {createModel, deleteModels, getModels, installAllPacks, validateModelStructure} from "./modules/data/data.js";
+import {defaultModels} from "./defaultModels.js";
+import {DefaultUserProvider} from "./providers.js";
+import formidableMiddleware from 'express-formidable';
+import sirv from "sirv";
+import * as tls from "node:tls";
+import {Event} from "./events.js";
+import path from "node:path";
+import { fileURLToPath } from 'node:url';
+// Constants
+
+// On définit __dirname pour obtenir le chemin absolu du répertoire courant,
+// ce qui est la méthode standard en ES Modules.
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+
+let caFile, certFile, keyFile;
+try {
+    if (process.env.CA_CERT)
+        caFile = fs.readFileSync(process.env.CA_CERT);
+} catch (e) {}
+try {
+    if (process.env.CERT)
+        certFile = fs.readFileSync(process.env.CERT);
+}catch (e) {}
+try{
+    if (process.env.CERT_KEY)
+        keyFile = fs.readFileSync(process.env.CERT_KEY);
+} catch (e) {}
+
+const secureContext = tls.createSecureContext({
+    ca: caFile, cert: certFile, key: keyFile
+});
+
+export const dbUrl = process.env.CI ? 'mongodb://mongodb:27017' : (process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017');
+
+const isTlsActive = !(!process.env.TLS || ["0", "false"].includes(process.env.TLS.toLowerCase()));
+
+const clientOptions = {
+    maxPoolSize: databasePoolSize
+};
+
+// We add TLS options if enabled
+if (isTlsActive) {
+    clientOptions.tls = true;
+
+    // is mTLS ? (client certificate required instead of password)
+    if (process.env.CERT) {
+        clientOptions.secureContext = tls.createSecureContext({
+            ca: fs.readFileSync(process.env.CA_CERT),
+            cert: fs.readFileSync(process.env.CERT),
+            key: fs.readFileSync(process.env.CERT_KEY)
+        });
+    }else {
+        // Path to the authority certificate
+        if (process.env.CA_CERT) {
+            clientOptions.tlsCAFile = process.env.CA_CERT;
+        }
+        // Path to the certificate key
+        if (process.env.CERT_KEY) {
+            clientOptions.tlsCertificateKeyFile = process.env.CERT_KEY;
+        }
+    }
+    if (tlsAllowInvalidCertificates) {
+        clientOptions.tlsAllowInvalidCertificates = true;
+        console.warn("🚨 [SECURITY WARNING] tlsAllowInvalidCertificates is ON. Server certificate will not be validated.");
+    }
+    if (tlsAllowInvalidHostnames) {
+        clientOptions.tlsAllowInvalidHostnames = true;
+        console.warn("🚨 [SECURITY WARNING] tlsAllowInvalidHostnames is ON. Server hostname will not be validated.");
+    }
+}
+
+export const MongoClient = new InternalMongoClient(dbUrl, clientOptions);
+
+
+// Database Name
+export const MongoDatabase = MongoClient.db(dbName);
+
+
+export const Engine = {
+    Create: async (options = { app : null}) => {
+        const engine = GameObject.Create("Engine");
+        console.log("Creating engine", Config.Get('modules'));
+        const logger = engine.addComponent(Logger);
+
+        engine.userProvider = new DefaultUserProvider(engine);
+
+        engine.setUserProvider = (providerInstance) => {
+            engine.userProvider = providerInstance;
+            logger.info(`Custom UserProvider '${providerInstance.constructor.name}' has been set.`);
+        };
+
+        if (!options.app) {
+            options.app = express();
+        }
+
+        const { app }  = options;
+        // Allows you to set port in the project properties.
+        app.set('port', process.env.PORT || 3000);
+        app.set('engine', engine);
+
+        app.use(formidableMiddleware({
+            encoding: 'utf-8',
+            uploadDir: process.cwd()+'/uploads/tmp',
+            multiples: true // req.files to be arrays of files
+        }));
+        app.use(cookieParser(process.env.COOKIES_SECRET || cookiesSecret));
+        app.use(requestIp.mw())
+
+        engine.use = (...args) => {
+            return app.use(...args);
+        }
+        engine.post = (...args) => {
+            return app.post(...args);
+        };
+        engine.get = (...args) => {
+            return app.get(...args);
+        };
+        engine.delete = (...args) => {
+            return app.delete(...args);
+        };
+        engine.patch = (...args) => {
+            return app.patch(...args);
+        };
+        engine.put = (...args) => {
+            return app.put(...args);
+        };
+        engine.all = (...args) => {
+            return app.all(...args);
+        };
+        engine.getModule = (module) => {
+            return engine._modules.find(m => m.module === module);
+        };
+
+        const importModule = async (module) => {
+            const moduleA = await import(module);
+            if (moduleA.onInit){
+                await moduleA.onInit(engine);
+                return {...moduleA, module};
+            }else {
+                const mod = moduleA.default();
+                await mod?.onInit(engine);
+                return { ...mod, module};
+            }
+        };
+
+        engine._modules = [];
+        for (const moduleIdentifier of Config.Get('modules', [])) {
+            try {
+                let moduleDir;
+                const moduleName = path.basename(moduleIdentifier);
+
+                const directPath = path.resolve(moduleIdentifier);
+                let isDir = fs.existsSync(directPath) && fs.statSync(directPath).isDirectory();
+
+                if (isDir) {
+                    moduleDir = directPath;
+                } else {
+                    moduleDir = path.resolve(__dirname, 'modules', moduleIdentifier);
+                }
+
+                let moduleEntryPoint;
+                const jsPath = moduleDir + '.js';
+                const indexJsPath = path.join(moduleDir, 'index.js');
+                const moduleJsPath = path.join(moduleDir, `${moduleName}.js`);
+
+                if (fs.existsSync(jsPath)) {
+                    moduleEntryPoint = 'file://' + jsPath;
+                } else if (fs.existsSync(indexJsPath)) {
+                    moduleEntryPoint = 'file://' + indexJsPath;
+                } else if (fs.existsSync(moduleJsPath)) {
+                    moduleEntryPoint = 'file://' + moduleJsPath;
+                }
+
+                if (moduleEntryPoint) {
+                    const loadedModule = await importModule(moduleEntryPoint);
+                    if (loadedModule) {
+                        engine._modules.push(loadedModule);
+                    }
+                } else {
+                    logger.warn(`Aucun point d'entrée trouvé pour le module '${moduleIdentifier}'.`);
+                }
+            } catch (e) {
+                logger.error(`Échec du chargement du module '${moduleIdentifier}':`, e.stack);
+            }
+        }
+
+        let server;
+
+        engine.start = async (port, cb) =>{
+            // Use connect method to connect to the server
+
+            // Start http server
+            server = http.createServer(app);
+
+            // Server Timeout Settings
+            server.timeout = 120000;
+            server.headersTimeout = 20000;
+            server.requestTimeout = 30000;
+            server.keepAliveTimeout = 5000;
+
+            server.listen(port);
+
+            await setupInitialModels();
+            await installAllPacks();
+
+            if (cb)
+                await cb();
+
+            engine.get('/api/health', (req, res) => {
+                res.status(200).json({
+                    status: 'ok',
+                    timestamp: new Date().toISOString()
+                });
+            });
+
+            app.use(sirv('client/dist', {
+                single: true,
+                dev: process.env.NODE_ENV === 'development'
+            }));
+
+            process.on('uncaughtException', function (exception) {
+                console.error(exception);
+                fs.appendFile('issues.txt', JSON.stringify({ code: exception.code, message: exception.message, stack: exception.stack }), function (err) {
+                    if (err){
+                        throw err;
+                    }
+                });
+                process.exit(1);
+            });
+
+            Event.Trigger("OnServerStart", "event", "system", engine);
+        }
+
+        engine.stop = async () => {
+            await server.close();
+            Event.Trigger("OnServerStop", "event", "system", engine);
+        };
+
+        async function setupInitialModels() {
+            logger.info("Validating structures of default models...");
+            const ms = Object.values(Config.Get('defaultModels', defaultModels));
+
+            let dbModels = await getModels();
+
+            for(let i = 0; i < ms.length; ++i){
+                const model = ms[i];
+                validateModelStructure(model);
+                // Création des modèles
+                if( !dbModels.find(m =>m.name === model.name) )
+                {
+                    model.locked = true;
+                    const r = await createModel(model);
+                    dbModels.push({...model, _id: r.insertedId });
+                    logger.info('Model inserted (' + model.name + ')');
+                }else
+                    logger.info('Model loaded (' + model.name + ')');
+            }
+            logger.info("All models loaded.");
+            Event.Trigger("OnModelsLoaded", "event", "system", engine, dbModels);
+        }
+        engine.resetModels = async () => {
+            await deleteModels();
+            Event.Trigger("OnModelsDeleted", "event", "system", engine);
+        };
+        return engine;
+    }
+}

package/src/modules/data/data.core.js

@@ -1,4 +1,7 @@
 import NodeCache from "node-cache";
+import i18n from "../../i18n.js";
+import {allowedFields, maxStringLength} from "../../constants.js";
+import path from "node:path";
 
 export const modelsCache = new NodeCache( { stdTTL: 100, checkperiod: 120 } );
 
@@ -15,3 +18,73 @@ export const mongoDBWhitelist = [
 ];
 export let importJobs = {};
 
+
+/**
+ * Exécute une tâche d'import/export (parsing, stringify) dans un worker thread.
+ * @param {('parse-json'|'parse-csv'|'stringify-json')} action - L'action à effectuer.
+ * @param {object} payload - Les données nécessaires pour l'action.
+ * @returns {Promise<any>} - Une promesse qui se résout avec les données traitées.
+ */
+export function runImportExportWorker(action, payload) {
+    return new Promise((resolve, reject) => {
+        const workerPath = path.resolve(process.cwd(), './src/workers/import-export-worker.js');
+        const worker = new Worker(workerPath);
+
+        worker.postMessage({ action, payload });
+
+        worker.on('message', (result) => {
+            if (result.success) {
+                resolve(result.data);
+            } else {
+                // Correction : On s'assure de toujours passer une chaîne de caractères à new Error()
+                const errorMessage = result.error || `Import/Export Worker failed with an unknown error. Action: ${action}.`;
+                reject(new Error(errorMessage));
+            }
+            worker.terminate();
+        });
+
+        worker.on('error', (err) => {
+            reject(err);
+            worker.terminate();
+        });
+
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Import/Export Worker stopped with exit code ${code}`));
+            }
+        });
+    });
+}
+/** Exécute une tâche de cryptographie dans un worker thread.
+ * @param {('encrypt'|'decrypt'|'hash')} action - L'action à effectuer.
+ * @param {object} payload - Les données nécessaires pour l'action.
+ * @returns {Promise<any>} - Une promesse qui se résout avec le résultat (si pertinent).
+ */
+export function runCryptoWorkerTask(action, payload) {
+    return new Promise((resolve, reject) => {
+        const workerPath = path.resolve(process.cwd(), './src/workers/crypto-worker.js');
+        const worker = new Worker(workerPath);
+
+        worker.postMessage({ action, payload });
+
+        worker.on('message', (result) => {
+            if (result.success) {
+                resolve(result.data); // Résout avec les données (ex: le hash) ou undefined si pas de retour
+            } else {
+                reject(new Error(result.error));
+            }
+            worker.terminate();
+        });
+
+        worker.on('error', (err) => {
+            reject(err);
+            worker.terminate();
+        });
+
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Crypto Worker stopped with exit code ${code}`));
+            }
+        });
+    });
+}

package/src/modules/data/data.js

@@ -78,7 +78,7 @@ import {getAllPacks} from "../../packs.js";
 import {Config} from "../../config.js";
 import {profiles} from "../../../client/src/constants.js";
 import {registerRoutes, sendSseToUser} from "./data.routes.js";
-import {importJobs, modelsCache, mongoDBWhitelist} from "./data.core.js";
+import {importJobs, modelsCache, mongoDBWhitelist, runCryptoWorkerTask, runImportExportWorker} from "./data.core.js";
 import readXlsxFile from "read-excel-file/node";
 
 let engine;
@@ -500,75 +500,6 @@ export const getAPILang = (langs) => {
     }, []).sort(((p,r) => p.quality < r.quality ? 1 : -1))?.[0].lang.split(/[-_]/)?.[0];
 }
 
-/**
- * Exécute une tâche d'import/export (parsing, stringify) dans un worker thread.
- * @param {('parse-json'|'parse-csv'|'stringify-json')} action - L'action à effectuer.
- * @param {object} payload - Les données nécessaires pour l'action.
- * @returns {Promise<any>} - Une promesse qui se résout avec les données traitées.
- */
-function runImportExportWorker(action, payload) {
-    return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(process.cwd(), './src/workers/import-export-worker.js');
-        const worker = new Worker(workerPath);
-
-        worker.postMessage({ action, payload });
-
-        worker.on('message', (result) => {
-            if (result.success) {
-                resolve(result.data);
-            } else {
-                // Correction : On s'assure de toujours passer une chaîne de caractères à new Error()
-                const errorMessage = result.error || `Import/Export Worker failed with an unknown error. Action: ${action}.`;
-                reject(new Error(errorMessage));
-            }
-            worker.terminate();
-        });
-
-        worker.on('error', (err) => {
-            reject(err);
-            worker.terminate();
-        });
-
-        worker.on('exit', (code) => {
-            if (code !== 0) {
-                reject(new Error(`Import/Export Worker stopped with exit code ${code}`));
-            }
-        });
-    });
-}
-/** Exécute une tâche de cryptographie dans un worker thread.
- * @param {('encrypt'|'decrypt'|'hash')} action - L'action à effectuer.
- * @param {object} payload - Les données nécessaires pour l'action.
- * @returns {Promise<any>} - Une promesse qui se résout avec le résultat (si pertinent).
- */
-function runCryptoWorkerTask(action, payload) {
-    return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(process.cwd(), './src/workers/crypto-worker.js');
-        const worker = new Worker(workerPath);
-
-        worker.postMessage({ action, payload });
-
-        worker.on('message', (result) => {
-            if (result.success) {
-                resolve(result.data); // Résout avec les données (ex: le hash) ou undefined si pas de retour
-            } else {
-                reject(new Error(result.error));
-            }
-            worker.terminate();
-        });
-
-        worker.on('error', (err) => {
-            reject(err);
-            worker.terminate();
-        });
-
-        worker.on('exit', (code) => {
-            if (code !== 0) {
-                reject(new Error(`Crypto Worker stopped with exit code ${code}`));
-            }
-        });
-    });
-}
 
 export function validateModelStructure(modelStructure) {
 
@@ -1353,6 +1284,7 @@ export async function onInit(defaultEngine) {
         packsCollection = getCollection("packs");
     }
     await registerRoutes(engine);
+    logger = engine.getComponent(Logger);
 
     // set backup scheduler
     schedule.scheduleJob("0 2 * * *", jobDumpUserData);
@@ -1364,8 +1296,6 @@ export async function onInit(defaultEngine) {
         dt.setTime(dt.getTime()-1000*3600*24*14);
         await deleteData("request", {"$lt": ["$timestamp",dt.toISOString()]}, null, false);
     });
-    await scheduleWorkflowTriggers();
-
     await scheduleAlerts();
 
 }

package/src/modules/data/data.routes.js

@@ -40,10 +40,10 @@ import {tutorialsConfig} from "../../../client/src/tutorials.js";
 import {
     cancelAlerts, deleteData,
     dumpUserData,
-    editData, exportData,
+    editData, editModel, exportData,
     getModel, getResource,
     handleCustomEndpointRequest,
-    handleDemoInitialization, importData, insertData, installPack,
+    handleDemoInitialization, importData, insertData, installPack, loadFromDump,
     patchData, searchData, validateModelStructure
 } from "./data.js";
 import process from "node:process";

package/src/modules/mongodb.js

@@ -10,15 +10,12 @@ export async function onInit(defaultEngine) {
     engine = defaultEngine;
     logger = engine.getComponent(Logger);
 
-    const isProduction = process.env.NODE_ENV === 'production'
-
-    modelsCollection = MongoDatabase.collection("models");
-    datasCollection = MongoDatabase.collection("datas");
-    filesCollection = MongoDatabase.collection("files");
-    packsCollection = MongoDatabase.collection("packs");
+    modelsCollection = getCollection("models");
+    datasCollection = getCollection("datas");
+    filesCollection = getCollection("files");
+    packsCollection = getCollection("packs");
 
     logger.info("MongoDB collections loaded.");
-
 };
 
 

package/src/modules/workflow.js

@@ -6,7 +6,7 @@ import crypto from "node:crypto";
 import ivm from 'isolated-vm';
 
 import {Logger} from "../gameObject.js";
-import {deleteData, insertData, patchData, searchData} from "./data/index.js";
+import {deleteData, insertData, patchData, scheduleAlerts, searchData} from "./data/index.js";
 import {emailDefaultConfig, maxExecutionsByStep, maxWorkflowSteps} from "../constants.js";
 import {ChatOpenAI} from "@langchain/openai";
 import {ChatGoogleGenerativeAI} from "@langchain/google-genai";
@@ -21,6 +21,8 @@ import util from "node:util";
 let logger = null;
 export async function onInit(defaultEngine) {
     logger = defaultEngine.getComponent(Logger);
+
+    await scheduleWorkflowTriggers();
 }
 
 /**

package/test/import_export.integration.test.js

@@ -48,7 +48,7 @@ function blobToFile(theBlob, fileName){
 }
 
 beforeAll(async () =>{
-    Config.Set("modules", ["data", "mongodb", "file", "bucket", "workflow","user", "assistant"]);
+    Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow","user", "assistant"]);
     await initEngine();
 })