data-primals-engine 1.4.3 → 1.5.1

package/src/modules/data/data.routes.js

@@ -4,14 +4,15 @@ import * as util from 'node:util';
 import {setTimeoutMiddleware} from '../../middlewares/timeout.js';
 import {isDemoUser, isLocalUser} from "../../data.js";
 import {
+    getHost,
     install,
     maxBytesPerSecondThrottleData,
     maxMagnetsDataPerModel,
     maxMagnetsModels,
-    maxModelsPerUser
+    maxModelsPerUser, maxPackData, maxPackPreviewData
 } from "../../constants.js";
 import {datasCollection, getCollection, getCollectionForUser, isObjectId, modelsCollection} from "../mongodb.js";
-import {safeAssignObject, uuidv4} from "../../core.js";
+import {countKeys, safeAssignObject, uuidv4} from "../../core.js";
 import {Event} from "../../events.js";
 import fs from "node:fs";
 import i18n from "../../i18n.js";
@@ -24,15 +25,16 @@ import {Config} from "../../config.js";
 import {processFilterPlaceholders} from "../../../client/src/filter.js";
 import {tutorialsConfig} from "../../../client/src/tutorials.js";
 import {getResource, handleDemoInitialization} from "./data.js";
-import process from "node:process";
+import process from "node:process"; 
 import {throttleMiddleware} from "../../middlewares/throttle.js";
 import {importJobs, modelsCache, runImportExportWorker} from "./data.core.js";
-import {validateModelStructure} from "./data.validation.js";
+import {validateModelData, validateModelStructure} from "./data.validation.js";
 import {
     deleteData,
     editData,
     editModel,
     exportData,
+    flushSearchCache,
     getModel,
     importData,
     insertData,
@@ -40,15 +42,13 @@ import {
     patchData,
     searchData
 } from "./data.operations.js";
-import {dumpUserData, loadFromDump} from "./data.backup.js";
+import { dumpUserData, loadFromDump } from "./data.backup.js";
+import { invalidateModelCache } from "./data.operations.js";
 
 let logger, engine;
 
 const sseConnections = new Map();
 
-const throttle = throttleMiddleware(maxBytesPerSecondThrottleData);
-
-
 
 
 async function logApiRequest(req, res, user, startTime, responseBody = null, error = null) {
@@ -146,6 +146,62 @@ export async function sendSseToUser(username, data) {
     return false;
 }
 
+/**
+ * Handles real-time validation for a single field or a subset of fields.
+ * This is designed to be called from the frontend with a debounce mechanism.
+ */
+export const validateDataRealtime = async (req, res) => {
+    // `data` is an object with the field(s) to validate, e.g., { "email": "test@example.com" }
+    // `contextId` is the _id of the document being edited, to exclude it from 'unique' checks.
+    const { model: modelName, data, contextId } = req.fields || req.body;
+    const user = req.me; // User is attached by middlewareAuthenticator
+
+    if (!modelName || !data || typeof data !== 'object' || Object.keys(data).length === 0) {
+        return res.status(400).json({ success: false, error: "Request must include 'model' and a non-empty 'data' object." });
+    }
+
+    const fieldName = Object.keys(data)[0]; // Assuming one field at a time for simplicity
+
+    try {
+        const model = await getModel(modelName, user);
+        if (!model) {
+            return res.status(404).json({ success: false, error: `Model '${modelName}' not found.` });
+        }
+
+        // 1. Use the existing validation logic (isPatch=true for partial validation)
+        await validateModelData(data, model, true, user);
+
+        // 2. Perform 'unique' checks against the database, which validateModelData might not do.
+        const fieldDefinition = model.fields.find(f => f.name === fieldName);
+        if (fieldDefinition && fieldDefinition.unique) {
+            const fieldValue = data[fieldName];
+
+            if (fieldValue !== undefined && fieldValue !== null && fieldValue !== '') {
+                const collection = await getCollectionForUser(user);
+                const query = {
+                    _user: user.username,
+                    _model: modelName,
+                    [fieldName]: fieldValue
+                };
+                
+                if (contextId && isObjectId(contextId)) {
+                    query._id = { $ne: new ObjectId(contextId) };
+                }
+
+                const existingDoc = await collection.findOne(query);
+                if (existingDoc) {
+                    throw new Error(i18n.t("api.data.duplicateValue", { field: fieldName, value: fieldValue }));
+                }
+            }
+        }
+
+        res.status(200).json({ success: true, message: "Validation successful.", field: fieldName });
+
+    } catch (error) {
+        res.status(400).json({ success: false, error: error.message, field: fieldName });
+    }
+};
+
 
 export async function handleCustomEndpointRequest(req, res) {
     const endpointDef = req.endpointDef;
@@ -254,12 +310,16 @@ export async function registerRoutes(defaultEngine){
     engine = defaultEngine;
     logger = engine.getComponent(Logger);
 
+    const m = Config.Get('maxBytesPerSecondThrottleData', maxBytesPerSecondThrottleData)
+    const throttle = throttleMiddleware(m);
+
     let userMiddlewares = await engine.userProvider.getMiddlewares();
 
     engine.all('/api/actions/:user/:path', [middlewareEndpointAuthenticator, userInitiator], handleCustomEndpointRequest);
     engine.all('/api/actions/:path', [middlewareAuthenticator, middlewareEndpointAuthenticator, userInitiator], handleCustomEndpointRequest);
     engine.post('/api/demo/initialize', [middlewareAuthenticator, userInitiator], handleDemoInitialization);
 
+    engine.post('/api/data/validate', [middlewareAuthenticator, userInitiator, middlewareLogger], validateDataRealtime);
     engine.post('/api/magnets', [middlewareAuthenticator, userInitiator], async (req, res) => {
         const user = req.me;
         const { name, description, modelNames } = req.fields; // Noms des modèles à inclure
@@ -357,7 +417,7 @@ export async function registerRoutes(defaultEngine){
             res.status(201).json({
                 success: true,
                 message: "Magnet link created successfully!",
-                url: `https://data.primals.net/magnet/${magnetUuid}` // ou votre URL de dev
+                url: `https://${getHost()}/magnet/${magnetUuid}` // ou votre URL de dev
             });
 
         } catch (error) {
@@ -445,16 +505,18 @@ export async function registerRoutes(defaultEngine){
             const { xpBonus, skill, achievement, notification } = tutorial.rewards;
 
             // --- LOGIQUE CORRIGÉE ---
-            let newData= {};
-            newData.xp = newData.xp || 0;
-            newData.achievements = newData.achievements || [];
-            newData.skills = newData.skills || [];
-            newData.completedTutorials = newData.completedTutorials || [];
+            // On part des données existantes de l'utilisateur pour ne rien écraser
+            let newData = {
+                xp: user.xp || 0,
+                achievements: [...(user.achievements || [])],
+                skills: JSON.parse(JSON.stringify(user.skills || [])), // Copie profonde pour éviter les mutations directes
+                completedTutorials: [...(user.completedTutorials || [])]
+            };
 
             // Appliquer les récompenses directement sur l'objet newData
             if (xpBonus) newData.xp += xpBonus;
             if (achievement && !newData.achievements.includes(achievement)) newData.achievements.push(achievement);
-            if (skill) {
+            if (skill && skill.name) { // S'assurer que la compétence a un nom
                 const existingSkill = newData.skills.find(s => s.name === skill.name);
                 if (existingSkill) {
                     existingSkill.points += skill.points;
@@ -868,13 +930,14 @@ export async function registerRoutes(defaultEngine){
 
         // get by name
         try {
+            const m = Config.Get('maxModelsPerUser', maxModelsPerUser);
             let models = await modelsCollection.find({$or: [{_user: {$exists: false}}]})
-                .sort({_user:-1, _id: 1 }).limit(maxModelsPerUser).toArray();
+                .sort({_user:-1, _id: 1 }).limit(m).toArray();
             models = models
                 .concat(
                     await modelsCollection.find({$or: [{_user: req.me._user}, {_user: req.me.username}]})
                         .sort({_user:-1, _id: 1 })
-                        .limit(maxModelsPerUser).toArray());
+                        .limit(m).toArray());
             res.json(models);
         } catch (error) {
             logger.error(error);
@@ -902,7 +965,8 @@ export async function registerRoutes(defaultEngine){
                 const count = await modelsCollection.count({
                     $and: [{_user: {$exists: true}}, {_user: req.me.username}]
                 });
-                if( count < maxModelsPerUser) {
+                const m = Config.Get('maxModelsPerUser', maxModelsPerUser);
+                if( count < m) {
                     if(await engine.userProvider.hasFeature(req.me, 'indexes')){
                         for (const field of modelData.fields) {
                             if( field.index ) {
@@ -987,6 +1051,9 @@ export async function registerRoutes(defaultEngine){
                 return res.status(404).json({error: i18n.t( "api.model.notFound", { model: modelName})});
             }
 
+            // Invalider le cache pour ce modèle avant de le supprimer
+            invalidateModelCache(modelName);
+
             if( await engine.userProvider.hasFeature(req.me, 'indexes') ) {
                 const indexes = await datasCollection.indexes();
                 for (const index of indexes) {
@@ -1479,11 +1546,11 @@ export async function registerRoutes(defaultEngine){
                 sortOptions['_updatedAt'] = -1; // Tri par défaut
             }
 
-            // On ne renvoie pas le champ 'datas' pour alléger la réponse de la liste
+            // On ne renvoie pas le champ 'data' pour alléger la réponse de la liste
             const packs = await packsCollection.find({
                 _user: req.query.user ? req.query.user : { $exists: false }
             }, {
-                projection: { datas: 0 }
+                projection: { data: 0 }
             }).sort(sortOptions).toArray();
 
             res.json(packs);
@@ -1508,8 +1575,33 @@ export async function registerRoutes(defaultEngine){
                 return res.status(404).json({ success: false, error: 'Pack not found.' });
             }
 
-            // On retourne le pack complet, incluant les données pour la vue de détail
-            res.json(pack);
+            const PACK_PREVIEW_LIMIT = Config.Get('maxPackPreviewData', maxPackPreviewData);
+
+            const countPackEntries = (p) => {
+                if (!p || !p.data) return 0;
+                let totalEntries = 0;
+                for (const langKey in p.data) {
+                    const langData = p.data[langKey];
+                    for (const modelKey in langData) {
+                        if (Array.isArray(langData[modelKey])) {
+                            totalEntries += langData[modelKey].length;
+                        }
+                    }
+                }
+                return totalEntries;
+            };
+
+            const totalEntries = countPackEntries(pack);
+
+            if (totalEntries > PACK_PREVIEW_LIMIT) {
+                logger.warn(`[GET /api/packs/${id}] Pack data is too large (${totalEntries} entries) and will be truncated for the client.`);
+                const packForClient = { ...pack };
+                delete packForClient.data;
+                packForClient.dataTruncated = true;
+                packForClient.totalDataEntries = totalEntries;
+                return res.json(packForClient);
+            }
+            res.json(pack); // On retourne le pack complet si sa taille est raisonnable
         } catch (error) {
             logger.error(`[GET /api/packs/${id}] Error fetching pack details:`, error);
             res.status(500).json({ success: false, error: 'Failed to fetch pack details.' });
@@ -1684,13 +1776,15 @@ export async function registerRoutes(defaultEngine){
         const user = req.me;
         const lang   = req.query.lang || req.fields.lang;
 
+        const packName = req.fields.packName || null;
+
         try {
             // Vérification des permissions
-            if (user.username !== 'demo' && isLocalUser(user) && !await hasPermission(["API_ADMIN", "API_INSTALL_PACK"], user)) {
+            if (!isDemoUser(user) && isLocalUser(user) && !await hasPermission(["API_ADMIN", "API_INSTALL_PACK"], user)) {
                 return res.status(403).json({ success: false, error: i18n.t('api.permission.installPack') });
             }
 
-            const result = await installPack({...req.fields.packData, private: true}, user, lang, { installForUser: true });
+            const result = await installPack(packName? packName : {...req.fields.packData, private: true}, user, lang, { installForUser: true });
 
             if (result.success) {
                 res.status(200).json({ success: true, message: `Pack installed successfully.`, summary: result.summary });

package/src/modules/data/data.scheduling.js

@@ -10,6 +10,7 @@ import {sendSseToUser} from "./data.routes.js";
 
 import {searchData} from "./data.operations.js";
 import {Logger} from "../../gameObject.js";
+import {Config} from "../../config.js";
 
 
 let engine, logger;
@@ -179,7 +180,7 @@ export async function scheduleAlerts() {
             // 4. Project (Slice): Pour chaque utilisateur, ne garder que les X premières alertes du tableau trié.
             {
                 $project: {
-                    oldestAlerts: {$slice: ["$alerts", maxAlertsPerUser]}
+                    oldestAlerts: {$slice: ["$alerts", Config.Get('maxAlertsPerUser', maxAlertsPerUser)]}
                 }
             },
             // 5. Unwind: Déconstruire le tableau 'oldestAlerts' pour obtenir un flux de documents, un par alerte.

package/src/modules/data/data.validation.js

@@ -2,6 +2,7 @@ import {Event} from "../../events.js";
 import i18n from "../../i18n.js";
 import {allowedFields, maxFileSize, maxModelNameLength, maxStringLength} from "../../constants.js";
 import {getDefaultForType} from "../../data.js";
+import { generateRegexFromMask } from './data.core.js';
 
 import {dataTypes} from "./data.operations.js";
 import {Logger} from "../../gameObject.js";
@@ -11,6 +12,63 @@ let engine, logger;
 export function onInit(defaultEngine) {
     engine = defaultEngine;
     logger = engine.getComponent(Logger);
+
+    Event.Listen("OnValidateModelStructure", async (modelStructure) =>{
+
+        const objectKeys = Object.keys(modelStructure);
+
+        if( objectKeys.find(o => !["name", "_user", "icon", "history", "locked", "_id", "description", "maxRequestData", "fields", "tags"].includes(o)) ){
+            throw new Error(i18n.t('api.model.invalidStructure'));
+        }
+
+        // Vérification du type de name
+        if (typeof modelStructure.name !== 'string' || !modelStructure.name) {
+            throw new Error(i18n.t("api.validate.requiredFieldString", ["name"]));
+        }
+
+        // Vérification du type de description
+        if (typeof modelStructure.description !== 'string') {
+            throw new Error(i18n.t("api.validate.fieldString", ["description"]));
+        }
+
+        // Vérification de la présence et du type du tableau fields
+        if (!Array.isArray(modelStructure.fields)) {
+            throw new Error(i18n.t('api.validate.fieldArray', ["fields"]));
+        }
+
+        // Vérification de la présence et du type du tableau fields
+        if (typeof(modelStructure.tags) !== 'undefined' && (!Array.isArray(modelStructure.tags) || modelStructure.tags.some(tag => typeof tag !== 'string'))) {
+            throw new Error(i18n.t('api.validate.fieldArray', ["tags"]));
+            //todo: fieldStringArray trad
+        }
+
+        // Vérification de chaque champ dans le tableau fields
+        for (const field of modelStructure.fields) {
+            validateField(field);
+        }
+
+        if (modelStructure.constraints) {
+            if (!Array.isArray(modelStructure.constraints)) {
+                throw new Error('Model "constraints" property must be an array.');
+            }
+            const fieldNames = new Set(modelStructure.fields.map(f => f.name));
+            for (const constraint of modelStructure.constraints) {
+                if (constraint.type === 'unique') {
+                    if (!constraint.name || !Array.isArray(constraint.keys) || constraint.keys.length === 0) {
+                        throw new Error('Unique constraint must have a "name" and a non-empty "keys" array.');
+                    }
+                    for (const key of constraint.keys) {
+                        if (!fieldNames.has(key)) {
+                            throw new Error(`Constraint key "${key}" in constraint "${constraint.name}" does not exist as a field in the model.`);
+                        }
+                    }
+                }
+            }
+        }
+
+        return true; // La structure du modèle est valide
+    }, "event", "system");
+
 }
 
 export async function validateModelStructure(modelStructure) {
@@ -104,17 +162,30 @@ export const validateField = (field) => {
     case 'code':
         if (field.type === 'code')
             allowedFieldTest(['maxlength', 'language', 'conditionBuilder', 'targetModel']);
-        else if (['string_t', 'string'].includes(field.type))
-            allowedFieldTest(['maxlength', 'multiline']);
+        else if (['string_t', 'string'].includes(field.type)) {
+            allowedFieldTest(['maxlength', 'multiline', 'mask', 'replacement']);
+        }
         else
             allowedFieldTest(['maxlength']);
         if (field.maxlength !== undefined && typeof field.maxlength !== 'number') {
             throw new Error(i18n.t('api.validate.fieldNumber', "L'attribut '{{0}}' doit être un nombre.", ["maxlength"]));
         }
+        if (field.mask !== undefined && typeof field.mask !== 'string') {
+            throw new Error(i18n.t('api.validate.fieldString', "Le champ '{{0}}' doit être une chaîne de caractères.", ["mask"]));
+        }
+        if (field.replacement !== undefined && typeof field.replacement !== 'object') {
+            throw new Error(i18n.t('api.validate.fieldObject', "L'attribut '{{0}}' doit être un objet.", ["replacement"]));
+        }
+        if (field.mask && !field.replacement) {
+            throw new Error(i18n.t('api.validate.missingField', "L'attribut 'replacement' est requis quand 'mask' est défini."));
+        }
         break;
     case 'model':
     case 'modelField':
-        allowedFieldTest([]);
+        allowedFieldTest(['targetModel']);
+        if (field.targetModel !== undefined && typeof field.targetModel !== 'string') {
+            throw new Error(i18n.t('api.validate.fieldString', "Le champ '{{0}}' doit être une chaîne de caractères.", ["targetModel"]));
+        }
         break;
     case 'object':
         allowedFieldTest([]);
@@ -238,6 +309,17 @@ export async function validateModelData(doc, model, isPatch = false) {
         const fieldDef = model.fields.find(f => f.name === fieldName);
         if (!fieldDef) continue; // On ignore les champs supplémentaires
 
+        // Validation du masque si défini
+        if (fieldDef.mask && value) {
+            const regexString = generateRegexFromMask(fieldDef.mask, fieldDef.replacement);
+            if (regexString) {
+                const regex = new RegExp(regexString);
+                if (!regex.test(value)) {
+                    throw new Error(i18n.t('api.field.maskValidationFailed', { field: fieldName, value: value, mask: fieldDef.mask }));
+                }
+            }
+        }
+
         const validator = dataTypes[fieldDef.type]?.validate;
         const valid = validator && validator(value, fieldDef);
         const realValidation = await Event.Trigger('OnDataValidate', "event", "system", value, fieldDef, doc);