data-primals-engine 1.1.1 → 1.1.3

package/client/src/constants.js

@@ -1,5 +1,5 @@
 
-export const seoTitle = 'Your data is here';
+export const seoTitle = 'Self Data Hosting';
 
 
 // Dans ConditionBuilder.jsx ou un fichier de constantes partagé
@@ -51,9 +51,9 @@ export const MONGO_OPERATORS = {
 
 export const profiles = {
     'personal': ['contact', 'location', 'imageGallery', 'budget', 'currency', 'taxonomy'], // budget,
-    'developer': ['alert','request','webpage', 'content', 'taxonomy', 'resource', 'translation', 'contact', 'location', 'channel', 'lang', 'token', 'message', 'ticket', 'user', 'permission', 'role'],
+    'developer': ['alert','endpoint','request','webpage', 'content', 'taxonomy', 'resource', 'translation', 'contact', 'location', 'channel', 'lang', 'token', 'message', 'ticket', 'user', 'permission', 'role'],
     'company': ['alert','request','location', 'campaign', 'order', 'currency', 'product', 'cart', 'cartItem', 'invoice', 'messaging', 'user', 'role', 'permission', 'token','translation', 'lang', 'webpage', 'content', 'taxonomy', 'contact', 'resource', 'accountingExercise', 'accountingLineItem', 'accountingEntry', 'employee', 'kpi', 'dashboard'],
-    'engineer': ['alert','request','dashboard', 'kpi', 'user', 'role', 'token', 'permission', 'workflow', 'workflowRun', 'workflowStep', "channel", "message", 'workflowAction', 'workflowTrigger']
+    'engineer': ['alert','endpoint','request','dashboard', 'kpi', 'user', 'role', 'token', 'permission', 'workflow', 'workflowRun', 'workflowStep', "channel", "message", 'workflowAction', 'workflowTrigger']
 }
 
 export const OPERAND_TYPES = {
@@ -64,5 +64,5 @@ export const OPERAND_TYPES = {
 
 
 export const getHost = () => {
-    return process.env.HOST || 'localhost';
+    return process.env.HOST || host || 'localhost';
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "data-primals-engine",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "data-primals-engine is the package responsible from handling large amount of data in a practical and performant way. It can handle large amount of data in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation.",
   "main": "src/engine.js",
   "type": "module",
@@ -90,7 +90,7 @@
     "tar": "^7.4.3",
     "uniqid": "^5.4.0",
     "vitest": "^3.2.4",
-    "vm2": "^3.9.19",
+    "isolated-vm": "^4.7.2",
     "yaml": "^2.8.0"
   },
   "devDependencies": {
@@ -115,6 +115,6 @@
   "author": "anonympins",
   "license": "MIT",
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   }
 }

package/server.js

@@ -25,17 +25,11 @@ if (process.argv.length === 3) {
     }
 }
 
+
 const port = process.env.PORT || 7633;
 engine.start(port, async (r) => {
     const logger = engine.getComponent(Logger);
     console.log("Server started on port" + port);
     timer.stop();
 
-    // 2. Use sirv to serve static files from 'client/dist'
-    // The 'single: true' option is key for SPAs. It will serve 'index.html'
-    // for any route that doesn't match a file, enabling client-side routing.
-    engine.use(sirv('client/dist', {
-        single: true,
-        dev: process.env.NODE_ENV === 'development'
-    }));
 });

package/src/constants.js

@@ -1,7 +1,32 @@
+/**
+ * Enables auto-installation at startup
+ * @type {boolean}
+ */
+export const install = true;
+
+/**
+ * Database name
+ * @type {string}
+ */
 export const dbName = "engine";
 
+/**
+ * Web server host (for cookie domain)
+ * @type {string}
+ */
+export const host = 'localhost'; // or myhost.tld
+
+/**
+ * Cookie secret key (if COOKIES_SECRET is set, it will override this variable)
+ * @type {string}
+ */
 export const cookiesSecret = 'hoaivuymzovyoznllmafivpzaovphlejvalwjvelfhqochakfesv';
 
+/**
+ * Available languages of the system
+ * You need to translate the i18n file, or translations ones.
+ * @type {string[]}
+ */
 export const availableLangs = [
     "en",
     "fa",
@@ -17,63 +42,180 @@ export const availableLangs = [
     "sv"
 ];
 
-
+/**
+ * AWS default configuration (overrided by AWS_* environment variables)
+ * @type {{bucketName: string, region: string}}
+ */
 export const awsDefaultConfig = {
     bucketName : 'bucket-primals',
     region: 'eu-north-1'
 }
 
+/**
+ * Email default configuration (overrided by SMTP_* environment variables)
+ * @type {{from: string}}
+ */
 export const emailDefaultConfig = {
-    from: "Support - data@primals.net <data@primals.net>"
+    from: "Support - data@primals.net <data@primals.net>",
+    host: 'smtp.mydomain.tld',
+    port: 2500,
+    secure: false,
+    user: 'user',
+    pass: 'password'
 }
 
-// 10000 tiny users
-// 1000 modern users
-// 100 mega utilisateurs potentiality
-// 250 000 entrées par utilisateur
+/**
+ * Maximum number of models per user
+ * @type {number}
+ */
 export const maxModelsPerUser = 1000;
+
+/**
+ * Maximum number of data per user
+ * @type {number}
+ */
 export const maxTotalDataPerUser = 500000;
-export const maxDataPerModelPerUser = 10000;
+
+/**
+ * Maximum length of a string
+ * @type {number}
+ */
 export const maxStringLength = 4096;
+
+/**
+ * Maximum length of a password
+ * @type {number}
+ */
 export const maxPasswordLength = 100000;
+
+/**
+ * Maximum length of a rich text
+ * @type {number}
+ */
 export const maxRichTextLength = 100000;
+
+/**
+ * Maximum number of exportable data per user per request
+ * @type {number}
+ */
 export const maxExportCount = 50000;
+
 export const maxMagnetsDataPerModel = 100;
 export const maxMagnetsModels = 20;
-export const defaultMaxRequestData = 500;
+
+/**
+ * Maximum number of data per request
+ * @type {number}
+ */
 export const maxRequestData = 2500;
+
+/**
+ * Maximum number of data per post
+ * @type {number}
+ */
 export const maxPostData = 500;
+
+
+/**
+ * Maximum number of relations per data
+ * @type {number}
+ */
 export const maxRelationsPerData = 1500;
-export const install = true;
+
+/**
+ * Maximum number of filters per request
+ * @type {number}
+ */
 export const maxFilterDepth = 8;
 export const elementsPerPage = 30;
 
-
+/**
+ * Maximum number of alerts per user
+ * @type {number}
+ */
 export const maxAlertsPerUser = 15;
+
+/**
+ * Storage safety margin (between 0 and 1)
+ * @type {number}
+ */
 export const storageSafetyMargin = 0.95;
+
+/**
+ * Number of bytes in Kilobytes constant
+ * @type {number}
+ */
 export const kilobytes = 1024;
+
+/**
+ * Number of bytes in Megabytes constant
+ * @type {number}
+ */
 export const megabytes = 1024*1024;
 
-export const maxBytesPerSecondThrottleFile = 800*kilobytes;  // 800ko/s
+/**
+ * Maximum bytes per second for data throttling
+ * @type {number}
+ */
 export const maxBytesPerSecondThrottleData = 200*kilobytes; // 200Ko/s
 
+/**
+ * Search request timeout (in milliseconds)
+ * @type {number}
+ */
 export const searchRequestTimeout = 15000;
 
-export const maxModelsInCache = 100000;
+/**
+ * Maximum model name length
+ * @type {number}
+ */
 export const maxModelNameLength = 150;
 
-export const maxDataSize = '20mb';
+/**
+ * Maximum file size (in bytes)
+ * @type {number}
+ */
 export const maxFileSize = 20 * 1024 * 1024; // 20 Mo
 
+/**
+ * Main fields types
+ * @type {string[]}
+ */
 export const mainFieldsTypes = ['string_t', 'string', 'url', 'enum', 'email', 'phone', 'date', 'datetime'];
 
+/**
+ * Maximum number of executions per step in workflows
+ * @type {number}
+ */
 export const maxExecutionsByStep = 5;
+/**
+ * Maximum number of steps per workflow
+ * @type {number}
+ */
 export const maxWorkflowSteps = 15;
 
+/**
+ * Maximum number of private files per user
+ * @type {number}
+ */
 export const maxPrivateFileSize = 20 * megabytes; // Taille max par fichier privé (20 Mo)
+
+/**
+ * Maximum total size of private files (in bytes)
+ * @type {number}
+ */
 export const maxTotalPrivateFilesSize = 250 * megabytes;
 
-//
+/**
+ * Timeout for Javacript execution in VMs (in milliseconds)
+ * @type {number}
+ */
+export const timeoutVM = 5000;
+
+/**
+ * Options for the HTML sanitizer
+ * @type {{allowedSchemesByTag: {}, selfClosing: string[], allowedSchemes: string[], enforceHtmlBoundary: boolean, disallowedTagsMode: string, allowProtocolRelative: boolean, allowedAttributes: {a: string[], img: string[], code: string[]}, allowedTags: string[], allowedSchemesAppliedToAttributes: string[]}}
+ */
 export const optionsSanitizer = {
     allowedTags: [
         "img",
@@ -102,8 +244,12 @@ export const optionsSanitizer = {
 }
 
 
+/**
+ * Meta models are arranging models in groups
+ * May evolve in the future with the use of packs
+ * @type {{}}
+ */
 export const metaModels = {};
-
 metaModels['common'] = { load: ['contact', 'location', 'request'] };
 metaModels['personal'] = { load: ['budget', 'imageGallery'] };
 metaModels['users'] = { load: ['permission', 'role', 'user', 'token'], 'require': ['i18n', 'common'] };
@@ -119,7 +265,10 @@ metaModels['workflow'] = { load: ['env', 'workflow', 'workflowRun', 'workflowAct
 metaModels['erp'] = { load: [ 'accountingExercise', 'accountingLineItem', 'accountingEntry', 'employee', 'dashboard', 'kpi'] };
 
 
-
+/**
+ * Available model field attributes
+ * @type {string[]}
+ */
 export const allowedFields = ['locked', 'hiddenable', 'anonymized', 'condition', 'color', 'index', 'type', 'required', 'hint', 'default', 'validate', 'unique', 'name', 'placeholder', 'asMain'];
 
 

package/src/defaultModels.js

@@ -1394,5 +1394,54 @@ export const defaultModels = {
                 "hint": "List of sponsors or partners involved in the event."
             }
         ]
+    },
+    endpoint: {
+        name: "endpoint",
+        description: "Defines custom API endpoints that execute a server-side script.",
+        fields: [
+            {
+                name: "name",
+                type: "string",
+                required: true,
+                asMain: true,
+                hint: "A human-readable name to identify the endpoint."
+            },
+            {
+                name: "isActive",
+                type: "boolean",
+                default: true,
+                hint: "If checked, the endpoint is active and can be called."
+            },
+            {
+                name: "path",
+                type: "string",
+                required: true,
+                hint: "The URL path after /api/actions/ (e.g., 'send-welcome-email'). Do not include '/'.",
+                placeholder: "my-custom-action"
+            },
+            {
+                name: "method",
+                type: "enum",
+                items: ["GET", "POST", "PUT", "PATCH", "DELETE"],
+                required: true,
+                default: "POST",
+                hint: "The HTTP method required to call this endpoint."
+            },
+            {
+                name: "code",
+                type: "code",
+                language: "javascript",
+                required: true,
+                hint: "The script to execute. Must return a value or an object that will be the JSON response.",
+                default: `// The script can access 'db', 'logger', 'env'.
+// request.body, request.query, request.params, request.headers available
+// The returned value will be the API's JSON response.
+
+logger.info('Custom endpoint executed with body:', request.body);
+
+return { success: true, message: 'Endpoint executed!', received: request.body };
+`
+            }
+        ]
     }
 };

package/src/email.js

@@ -50,7 +50,7 @@ export const sendEmail = async (email = "", data, smtpConfig = null, lang, tpl =
     if (emails.length === 0) return;
 
     // Choisir le transporteur à utiliser
-    const transporter = smtpConfig ? createTransporter(smtpConfig) : defaultTransporter;
+    const transporter = smtpConfig ? createTransporter(smtpConfig||emailDefaultConfig) : defaultTransporter;
 
     if (tpl === null) tpl = event_trigger("sendEmail:template",data, lang);
     let html = tpl;

package/src/engine.js

@@ -12,6 +12,7 @@ import {createModel, deleteModels, getModels, validateModelStructure} from "./mo
 import {defaultModels} from "./defaultModels.js";
 import {DefaultUserProvider} from "./providers.js";
 import formidableMiddleware from 'express-formidable';
+import sirv from "sirv";
 
 // Constants
 const isProduction = process.env.NODE_ENV === 'production'
@@ -37,6 +38,7 @@ export const Engine = {
             engine.getComponent(Logger).info(`Custom UserProvider '${providerInstance.constructor.name}' has been set.`);
         };
 
+
         const app = express();
         // Allows you to set port in the project properties.
         app.set('port', process.env.PORT || 3000);
@@ -68,6 +70,9 @@ export const Engine = {
         engine.put = (...args) => {
             return app.put(...args);
         };
+        engine.all = (...args) => {
+            return app.all(...args);
+        };
         engine.getModule = (module) => {
             return engine._modules.find(m => m.module === module);
         };
@@ -102,6 +107,12 @@ export const Engine = {
             return Promise.resolve();
         });
         let server;
+
+        app.use(sirv('client/dist', {
+            single: true,
+            dev: process.env.NODE_ENV === 'development'
+        }));
+
         engine.start = async (port, cb) =>{
             // Use connect method to connect to the server
 
@@ -160,6 +171,12 @@ export const Engine = {
         engine.resetModels = async () => {
             await deleteModels();
         };
+        engine.get('/api/health', (req, res) => {
+            res.status(200).json({
+                status: 'ok',
+                timestamp: new Date().toISOString()
+            });
+        });
         return engine;
     }
 }

package/src/modules/data.js

@@ -66,6 +66,7 @@ import schedule from "node-schedule";
 import {middleware} from "../middlewares/middleware-mongodb.js";
 import i18n from "data-primals-engine/i18n";
 import {
+    executeSafeJavascript,
     runScheduledJobWithDbLock,
     scheduleWorkflowTriggers,
     triggerWorkflows
@@ -87,6 +88,7 @@ import {assistantGlobalLimiter} from "./assistant.js";
 import {getAllPacks} from "../packs.js";
 import {throttleMiddleware} from "../middlewares/throttle.js";
 import {Config} from "../config.js";
+import {profiles} from "../../client/src/constants.js";
 
 // Obtenir le chemin du répertoire courant de manière fiable avec ES Modules
 const __filename = fileURLToPath(import.meta.url);
@@ -1216,6 +1218,80 @@ export const editModel = async (user, id, data) => {
 };
 
 
+export async function handleCustomEndpointRequest(req, res) {
+    const { path } = req.params;
+    const method = req.method.toUpperCase();
+
+    const user = req.me;
+    if (!user) {
+        return res.status(401).json({ success: false, message: 'Authentication required.' });
+    }
+
+    try {
+        // 1. Trouver l'endpoint correspondant dans la base de données
+        const endpointSearch = await searchData({
+            user,
+            query: {
+                model: 'endpoint',
+                filter: {
+                    path: path,
+                    method: method,
+                    isActive: true
+                },
+                limit: 1
+            }
+        });
+
+        if (endpointSearch.count === 0) {
+            logger.warn(`[Endpoint] 404 - No active endpoint found for user '${user.username}', path '${path}', method '${method}'.`);
+            return res.status(404).json({ success: false, message: 'Endpoint not found.' });
+        }
+
+        const endpointDef = endpointSearch.data[0];
+
+        // 2. Préparer le contexte pour le script
+        // On donne au script accès au corps, aux paramètres de la requête, etc.
+        const contextData = {
+            request:{
+                body: req.fields,
+                query: req.query,
+                params: req.params,
+                headers: req.headers
+            }
+        };
+
+        // 3. Exécuter le code de l'endpoint en utilisant notre sandbox sécurisé
+        logger.info(`[Endpoint] Executing endpoint '${endpointDef.name}' for user '${user.username}'.`);
+        const result = await executeSafeJavascript(
+            { script: endpointDef.code }, // On passe la définition du script
+            contextData,
+            user
+        );
+
+        // 4. Envoyer la réponse
+        if (result.success) {
+            // Le script a réussi, on retourne sa sortie
+            res.status(200).json(result.data);
+        } else {
+            // Le script a échoué, on retourne une erreur 500 avec les logs
+            logger.error(`[Endpoint] Execution failed for '${endpointDef.name}'. Error: ${result.message}`);
+
+            const r = {
+                success: false,
+                message: 'Endpoint script execution failed.',
+                // On peut choisir d'exposer les logs pour le débogage
+                details: result.message
+            };
+            r.logs = result.logs;
+            res.status(500).json(r);
+        }
+
+    } catch (error) {
+        logger.error(`[Endpoint] Critical error handling request for path '${path}': ${error.message}`, process.env.NODE_ENV === 'development'? error.stack : error.stack[0]);
+        res.status(500).json({ success: false, message: 'An internal server error occurred.' });
+    }
+}
+
 export async function onInit(defaultEngine) {
     engine = defaultEngine;
     logger = engine.getComponent(Logger);
@@ -1322,18 +1398,9 @@ export async function onInit(defaultEngine) {
     await scheduleWorkflowTriggers();
 
     await scheduleAlerts();
-    // Dans onInit(defaultEngine) { ... }
-    // ...
 
-    const saveUser = async (user, data) => {
-
-        const primalsDb = MongoClient.db("primals");
-
-        let usersCollection = primalsDb.collection("users");
-        return await usersCollection.updateOne({ username: user.username }, { $set: data }, { upsert: true })
-    }
-
-    // Dans C:/Dev/hackersonline-engine/server/src/modules/data.js, dans onInit()
+    engine.all('/api/actions/:path', [middlewareAuthenticator, userInitiator], handleCustomEndpointRequest);
+    engine.post('/api/demo/initialize', [middlewareAuthenticator, userInitiator], handleDemoInitialization);
 
     engine.post('/api/magnets', [middlewareAuthenticator, userInitiator], async (req, res) => {
         const user = req.me;
@@ -2567,7 +2634,7 @@ export async function onInit(defaultEngine) {
                 return res.status(403).json({ success: false, error: i18n.t('api.permission.installPack') });
             }
 
-            const result = await installPack(logger, id, user, lang);
+            const result = await installPack(id, user, lang);
 
             if (result.success) {
                 res.status(200).json({ success: true, message: `Pack installed successfully.`, summary: result.summary });
@@ -5365,7 +5432,7 @@ async function logApiRequest(req, res, user, startTime, responseBody = null, err
  * @param {string} lang - Le code de langue pour les données spécifiques.
  * @returns {Promise<{success: boolean, summary: object, errors: Array, modifiedCount: number}>}
  */
-export async function installPack(logger, packId, user, lang) {
+export async function installPack(packId, user, lang) {
     const packsCollection = getCollection('packs');
     const pack = await packsCollection.findOne({ _id: new ObjectId(packId) });
 
@@ -5603,4 +5670,70 @@ export const installAllPacks = async () => {
     console.log(util.inspect(packs, false, 20, true));
     await packsCollection.deleteMany({ _user: { $exists : false }});
     await packsCollection.insertMany(packs);
+}
+
+// Dans C:/Dev/data-primals-engine/src/modules/data.js
+// Dans C:/Dev/data-primals-engine/src/modules/data.js
+
+// ... (imports inchangés)
+
+export async function handleDemoInitialization(req, res) {
+    const user = req.me;
+    const body = req.fields;
+    const models = (Object.keys(profiles).includes(body.profile) && profiles[body.profile]) || '';
+    if (!isDemoUser(user)) {
+        return res.status(403).json({ success: false, error: "This action is only for demo users." });
+    }
+    if (!Array.isArray(models) || models.length === 0) {
+        return res.status(400).json({ success: false, error: "A valid 'models' array is required." });
+    }
+
+    logger.info(`[Demo Init] Starting initialization for user '${user.username}' with ${models.length} models.`);
+
+    try {
+        // 1. Nettoyage de l'environnement (inchangé)
+        const datasCollection = getCollection("datas");
+        const modelsCollection = getCollection("models");
+        const filesCollection = getCollection("files");
+
+        await datasCollection.deleteMany({ _user: user.username });
+        await modelsCollection.deleteMany({ _user: user.username });
+        const files = await filesCollection.find({ user: user.username }).toArray();
+        for (const file of files) {
+            await removeFile(file.guid, user).catch(e => logger.error(e.message));
+        }
+        await cancelAlerts(user);
+        logger.info(`[Demo Init] Environment cleaned for user '${user.username}'.`);
+
+        const packToInstall = {
+            name: `dynamic-pack-for-${user.username}-${Date.now()}`,
+            description: `Dynamically generated pack for profile models.`,
+            models: models,
+            data: {}
+        };
+
+        logger.info(`[Demo Init] Installing dynamically generated pack with models: [${models.join(', ')}].`);
+
+        // Create and install pack
+        const packsCollection = getCollection('packs');
+        const tempPack = { ...packToInstall, _user: 'system_temp' };
+        const tempInsert = await packsCollection.insertOne(tempPack);
+        const packId = tempInsert.insertedId;
+
+        const result = await installPack(packId, user, req.query.lang || 'en');
+
+        await packsCollection.deleteOne({ _id: packId });
+
+        if (result.success || result.modifiedCount > 0) {
+            logger.info(`[Demo Init] Pack installed successfully for user '${user.username}'.`);
+            res.status(200).json({ success: true, message: "Demo environment initialized successfully.", summary: result.summary });
+        } else {
+            logger.error(`[Demo Init] Pack installation failed for user '${user.username}'.`);
+            res.status(500).json({ success: false, error: 'Demo pack installation failed.', errors: result.errors });
+        }
+
+    } catch (error) {
+        logger.error(`[Demo Init] Critical error during initialization for user '${user.username}':`, error);
+        res.status(500).json({ success: false, error: 'An internal server error occurred during initialization.' });
+    }
 }