data-primals-engine 1.2.6-rc2 → 1.2.6-rc3

package/README.md

@@ -315,7 +315,7 @@ curl -X DELETE http://localhost:7633/api/data?_user=demo \
 Make sure you use the code below to initialize the user : 
 ```javascript
 import express from "express";
-import {Engine, insertData, searchData } from './src/index';
+import {Engine, insertData, searchData } from 'data-primals-engine';
 
 // Ensure the engine is initialized
 

package/package.json

@@ -1,120 +1,120 @@
-{
-  "name": "data-primals-engine",
-  "version": "1.2.6-rc2",
-  "description": "data-primals-engine is a package responsible from handling large amount of data using MongoDB in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation. It also has integrated AI assistant.",
-  "main": "src/engine.js",
-  "type": "module",
-  "scripts": {
-    "preinstall": "npx force-resolutions",
-    "dev": "concurrently \"npm:client\" \"npm:devserver\"",
-    "prod": "npm run build-server && npm run server",
-    "client": "cd client && npm run dev",
-    "build-server": "vite build client --config client/vite.config.js --outDir dist",
-    "build-client": "cd client && npm run build",
-    "server": "cross-env NODE_ENV=production node server.js",
-    "devserver": "npm run build-server && cross-env NODE_ENV=development PORT=7633 node server.js",
-    "lint": "eslint .",
-    "format": "prettier --write .",
-    "clean": "rm -rf node_modules package-lock.json",
-    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
-    "audit": "npm audit --audit-level=high",
-    "migrate:create": "node src/migrate.js create",
-    "migrate:up": "node src/migrate.js up",
-    "migrate:down": "node src/migrate.js down",
-    "migrate:revert": "node src/migrate.js revert",
-    "migrate:status": "node src/migrate.js status"
-  },
-  "optionalDependencies": {
-    "@rollup/rollup-linux-x64-gnu": "4.6.1"
-  },
-  "resolutions": {
-    "tar-fs": "3.0.9",
-    "on-headers": "1.1.0",
-    "brace-expansion": "2.0.2",
-    "prismjs": "1.30.0"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/anonympins/data-primals-engine.git"
-  },
-  "exports": {
-    ".": "./src/index.js",
-    "./modules/*.js": "./src/modules/*.js",
-    "./modules/*": "./src/modules/*/index.js",
-    "./client": "./client/index.js",
-    "./*": "./src/*.js"
-  },
-  "peerDependencies": {
-    "express": "^5.1.0",
-    "react": ">=18.0.0",
-    "react-query": ">=3.0.0"
-  },
-  "dependencies": {
-    "@langchain/core": "^0.3.66",
-    "@langchain/deepseek": "^0.1.0",
-    "@langchain/google-genai": "^0.2.16",
-    "@langchain/openai": "^0.6.3",
-    "archiver": "^7.0.1",
-    "aws-sdk": "^2.1692.0",
-    "bcrypt": "^6.0.0",
-    "body-parser": "^2.2.0",
-    "chalk": "^5.4.1",
-    "check-disk-space": "^3.4.0",
-    "compression": "^1.8.1",
-    "cookie-parser": "^1.4.7",
-    "cronstrue": "^3.2.0",
-    "csv-parse": "^6.1.0",
-    "date-fns": "^4.1.0",
-    "express-csrf-double-submit-cookie": "^2.0.0",
-    "express-formidable": "^1.2.0",
-    "express-mongo-sanitize": "^2.2.0",
-    "express-rate-limit": "^8.0.1",
-    "express-session": "^1.18.2",
-    "i18next-browser-languagedetector": "^8.2.0",
-    "isolated-vm": "^4.7.2",
-    "juice": "^11.0.1",
-    "mathjs": "^14.6.0",
-    "mongodb": "^6.18.0",
-    "node-cache": "^5.1.2",
-    "node-schedule": "^2.1.1",
-    "nodemailer": "^7.0.5",
-    "openai": "^5.10.2",
-    "process": "^0.11.10",
-    "prop-types": "^15.8.1",
-    "randomcolor": "^0.6.2",
-    "react-i18next": "^15.6.1",
-    "react-markdown": "^10.1.0",
-    "read-excel-file": "^5.8.8",
-    "request-ip": "^3.3.0",
-    "sanitize-html": "^2.17.0",
-    "sirv": "^3.0.1",
-    "swagger-ui-express": "^5.0.1",
-    "tar": "^7.4.3",
-    "uniqid": "^5.4.0",
-    "vitest": "^3.2.4",
-    "yaml": "^2.8.0"
-  },
-  "devDependencies": {
-    "@eslint/js": "^9.32.0",
-    "concurrently": "^9.2.0",
-    "cross-env": "^10.0.0",
-    "eslint": "^9.32.0",
-    "globals": "^16.3.0"
-  },
-  "subPackages": [
-    "client"
-  ],
-  "keywords": [
-    "data-driven engine",
-    "headless CMS",
-    "backend",
-    "automation",
-    "AWS S3",
-    "MongoDB"
-  ],
-  "author": "anonympins",
-  "license": "MIT",
-  "engines": {
-    "node": ">=20.0.0"
-  }
-}
+{
+  "name": "data-primals-engine",
+  "version": "1.2.6-rc3",
+  "description": "data-primals-engine is a package responsible from handling large amount of data using MongoDB in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation. It also has integrated AI assistant.",
+  "main": "src/engine.js",
+  "type": "module",
+  "scripts": {
+    "preinstall": "npx force-resolutions",
+    "dev": "concurrently \"npm:client\" \"npm:devserver\"",
+    "prod": "npm run build-server && npm run server",
+    "client": "cd client && npm run dev",
+    "build-server": "vite build client --config client/vite.config.js --outDir dist",
+    "build-client": "cd client && npm run build",
+    "server": "cross-env NODE_ENV=production node server.js",
+    "devserver": "npm run build-server && cross-env NODE_ENV=development PORT=7633 node server.js",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "clean": "rm -rf node_modules package-lock.json",
+    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
+    "audit": "npm audit --audit-level=high",
+    "migrate:create": "node src/migrate.js create",
+    "migrate:up": "node src/migrate.js up",
+    "migrate:down": "node src/migrate.js down",
+    "migrate:revert": "node src/migrate.js revert",
+    "migrate:status": "node src/migrate.js status"
+  },
+  "optionalDependencies": {
+    "@rollup/rollup-linux-x64-gnu": "4.6.1"
+  },
+  "resolutions": {
+    "tar-fs": "3.0.9",
+    "on-headers": "1.1.0",
+    "brace-expansion": "2.0.2",
+    "prismjs": "1.30.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/anonympins/data-primals-engine.git"
+  },
+  "exports": {
+    ".": "./src/index.js",
+    "./modules/*.js": "./src/modules/*.js",
+    "./modules/*": "./src/modules/*/index.js",
+    "./client": "./client/index.js",
+    "./*": "./src/*.js"
+  },
+  "peerDependencies": {
+    "express": "^5.1.0",
+    "react": ">=18.0.0",
+    "react-query": ">=3.0.0"
+  },
+  "dependencies": {
+    "@langchain/core": "^0.3.66",
+    "@langchain/deepseek": "^0.1.0",
+    "@langchain/google-genai": "^0.2.16",
+    "@langchain/openai": "^0.6.3",
+    "archiver": "^7.0.1",
+    "aws-sdk": "^2.1692.0",
+    "bcrypt": "^6.0.0",
+    "body-parser": "^2.2.0",
+    "chalk": "^5.4.1",
+    "check-disk-space": "^3.4.0",
+    "compression": "^1.8.1",
+    "cookie-parser": "^1.4.7",
+    "cronstrue": "^3.2.0",
+    "csv-parse": "^6.1.0",
+    "date-fns": "^4.1.0",
+    "express-csrf-double-submit-cookie": "^2.0.0",
+    "express-formidable": "^1.2.0",
+    "express-mongo-sanitize": "^2.2.0",
+    "express-rate-limit": "^8.0.1",
+    "express-session": "^1.18.2",
+    "i18next-browser-languagedetector": "^8.2.0",
+    "isolated-vm": "^4.7.2",
+    "juice": "^11.0.1",
+    "mathjs": "^14.6.0",
+    "mongodb": "^6.18.0",
+    "node-cache": "^5.1.2",
+    "node-schedule": "^2.1.1",
+    "nodemailer": "^7.0.5",
+    "openai": "^5.10.2",
+    "process": "^0.11.10",
+    "prop-types": "^15.8.1",
+    "randomcolor": "^0.6.2",
+    "react-i18next": "^15.6.1",
+    "react-markdown": "^10.1.0",
+    "read-excel-file": "^5.8.8",
+    "request-ip": "^3.3.0",
+    "sanitize-html": "^2.17.0",
+    "sirv": "^3.0.1",
+    "swagger-ui-express": "^5.0.1",
+    "tar": "^7.4.3",
+    "uniqid": "^5.4.0",
+    "vitest": "^3.2.4",
+    "yaml": "^2.8.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.32.0",
+    "concurrently": "^9.2.0",
+    "cross-env": "^10.0.0",
+    "eslint": "^9.32.0",
+    "globals": "^16.3.0"
+  },
+  "subPackages": [
+    "client"
+  ],
+  "keywords": [
+    "data-driven engine",
+    "headless CMS",
+    "backend",
+    "automation",
+    "AWS S3",
+    "MongoDB"
+  ],
+  "author": "anonympins",
+  "license": "MIT",
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/src/engine.js

@@ -199,7 +199,6 @@ export const Engine = {
                 }
             } catch (e) {
                 logger.error(`Échec du chargement du module '${moduleIdentifier}':`, e.stack);
-                return null;
             }
         }
 

package/src/modules/data/data.core.js

@@ -1,4 +1,7 @@
 import NodeCache from "node-cache";
+import i18n from "../../i18n.js";
+import {allowedFields, maxStringLength} from "../../constants.js";
+import path from "node:path";
 
 export const modelsCache = new NodeCache( { stdTTL: 100, checkperiod: 120 } );
 
@@ -15,3 +18,73 @@ export const mongoDBWhitelist = [
 ];
 export let importJobs = {};
 
+
+/**
+ * Exécute une tâche d'import/export (parsing, stringify) dans un worker thread.
+ * @param {('parse-json'|'parse-csv'|'stringify-json')} action - L'action à effectuer.
+ * @param {object} payload - Les données nécessaires pour l'action.
+ * @returns {Promise<any>} - Une promesse qui se résout avec les données traitées.
+ */
+export function runImportExportWorker(action, payload) {
+    return new Promise((resolve, reject) => {
+        const workerPath = path.resolve(process.cwd(), './src/workers/import-export-worker.js');
+        const worker = new Worker(workerPath);
+
+        worker.postMessage({ action, payload });
+
+        worker.on('message', (result) => {
+            if (result.success) {
+                resolve(result.data);
+            } else {
+                // Correction : On s'assure de toujours passer une chaîne de caractères à new Error()
+                const errorMessage = result.error || `Import/Export Worker failed with an unknown error. Action: ${action}.`;
+                reject(new Error(errorMessage));
+            }
+            worker.terminate();
+        });
+
+        worker.on('error', (err) => {
+            reject(err);
+            worker.terminate();
+        });
+
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Import/Export Worker stopped with exit code ${code}`));
+            }
+        });
+    });
+}
+/** Exécute une tâche de cryptographie dans un worker thread.
+ * @param {('encrypt'|'decrypt'|'hash')} action - L'action à effectuer.
+ * @param {object} payload - Les données nécessaires pour l'action.
+ * @returns {Promise<any>} - Une promesse qui se résout avec le résultat (si pertinent).
+ */
+export function runCryptoWorkerTask(action, payload) {
+    return new Promise((resolve, reject) => {
+        const workerPath = path.resolve(process.cwd(), './src/workers/crypto-worker.js');
+        const worker = new Worker(workerPath);
+
+        worker.postMessage({ action, payload });
+
+        worker.on('message', (result) => {
+            if (result.success) {
+                resolve(result.data); // Résout avec les données (ex: le hash) ou undefined si pas de retour
+            } else {
+                reject(new Error(result.error));
+            }
+            worker.terminate();
+        });
+
+        worker.on('error', (err) => {
+            reject(err);
+            worker.terminate();
+        });
+
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Crypto Worker stopped with exit code ${code}`));
+            }
+        });
+    });
+}

package/src/modules/data/data.js

@@ -78,7 +78,7 @@ import {getAllPacks} from "../../packs.js";
 import {Config} from "../../config.js";
 import {profiles} from "../../../client/src/constants.js";
 import {registerRoutes, sendSseToUser} from "./data.routes.js";
-import {importJobs, modelsCache, mongoDBWhitelist} from "./data.core.js";
+import {importJobs, modelsCache, mongoDBWhitelist, runCryptoWorkerTask, runImportExportWorker} from "./data.core.js";
 import readXlsxFile from "read-excel-file/node";
 
 let engine;
@@ -500,75 +500,6 @@ export const getAPILang = (langs) => {
     }, []).sort(((p,r) => p.quality < r.quality ? 1 : -1))?.[0].lang.split(/[-_]/)?.[0];
 }
 
-/**
- * Exécute une tâche d'import/export (parsing, stringify) dans un worker thread.
- * @param {('parse-json'|'parse-csv'|'stringify-json')} action - L'action à effectuer.
- * @param {object} payload - Les données nécessaires pour l'action.
- * @returns {Promise<any>} - Une promesse qui se résout avec les données traitées.
- */
-function runImportExportWorker(action, payload) {
-    return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(process.cwd(), './src/workers/import-export-worker.js');
-        const worker = new Worker(workerPath);
-
-        worker.postMessage({ action, payload });
-
-        worker.on('message', (result) => {
-            if (result.success) {
-                resolve(result.data);
-            } else {
-                // Correction : On s'assure de toujours passer une chaîne de caractères à new Error()
-                const errorMessage = result.error || `Import/Export Worker failed with an unknown error. Action: ${action}.`;
-                reject(new Error(errorMessage));
-            }
-            worker.terminate();
-        });
-
-        worker.on('error', (err) => {
-            reject(err);
-            worker.terminate();
-        });
-
-        worker.on('exit', (code) => {
-            if (code !== 0) {
-                reject(new Error(`Import/Export Worker stopped with exit code ${code}`));
-            }
-        });
-    });
-}
-/** Exécute une tâche de cryptographie dans un worker thread.
- * @param {('encrypt'|'decrypt'|'hash')} action - L'action à effectuer.
- * @param {object} payload - Les données nécessaires pour l'action.
- * @returns {Promise<any>} - Une promesse qui se résout avec le résultat (si pertinent).
- */
-function runCryptoWorkerTask(action, payload) {
-    return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(process.cwd(), './src/workers/crypto-worker.js');
-        const worker = new Worker(workerPath);
-
-        worker.postMessage({ action, payload });
-
-        worker.on('message', (result) => {
-            if (result.success) {
-                resolve(result.data); // Résout avec les données (ex: le hash) ou undefined si pas de retour
-            } else {
-                reject(new Error(result.error));
-            }
-            worker.terminate();
-        });
-
-        worker.on('error', (err) => {
-            reject(err);
-            worker.terminate();
-        });
-
-        worker.on('exit', (code) => {
-            if (code !== 0) {
-                reject(new Error(`Crypto Worker stopped with exit code ${code}`));
-            }
-        });
-    });
-}
 
 export function validateModelStructure(modelStructure) {
 

package/src/modules/data/data.routes.js

@@ -40,10 +40,10 @@ import {tutorialsConfig} from "../../../client/src/tutorials.js";
 import {
     cancelAlerts, deleteData,
     dumpUserData,
-    editData, exportData,
+    editData, editModel, exportData,
     getModel, getResource,
     handleCustomEndpointRequest,
-    handleDemoInitialization, importData, insertData, installPack,
+    handleDemoInitialization, importData, insertData, installPack, loadFromDump,
     patchData, searchData, validateModelStructure
 } from "./data.js";
 import process from "node:process";