data-primals-engine 1.1.3 → 1.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "data-primals-engine",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "data-primals-engine is the package responsible from handling large amount of data in a practical and performant way. It can handle large amount of data in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation.",
   "main": "src/engine.js",
   "type": "module",
@@ -44,7 +44,8 @@
   },
   "peerDependencies": {
     "react": ">=18.0.0",
-    "react-query": ">=3.0.0"
+    "react-query": ">=3.0.0",
+    "express": "^5.1.0"
   },
   "dependencies": {
     "@langchain/core": "^0.3.66",
@@ -60,15 +61,13 @@
     "cookie-parser": "^1.4.7",
     "cronstrue": "^3.2.0",
     "csv-parse": "^6.1.0",
-    "data-primals-engine": "^1.0.11",
+    "data-primals-engine": "^1.0.14",
     "date-fns": "^4.1.0",
-    "express": "^5.1.0",
     "express-csrf-double-submit-cookie": "^2.0.0",
     "express-formidable": "^1.2.0",
     "express-mongo-sanitize": "^2.2.0",
     "express-rate-limit": "^8.0.1",
     "express-session": "^1.18.2",
-    "i18next": "^25.3.2",
     "i18next-browser-languagedetector": "^8.2.0",
     "juice": "^11.0.1",
     "mathjs": "^14.6.0",
@@ -80,7 +79,6 @@
     "process": "^0.11.10",
     "prop-types": "^15.8.1",
     "randomcolor": "^0.6.2",
-    "react-helmet": "^6.1.0",
     "react-i18next": "^15.6.1",
     "react-markdown": "^10.1.0",
     "request-ip": "^3.3.0",

package/server.js

@@ -6,6 +6,7 @@
 import process from "node:process";
 import {Config, Engine, BenchmarkTool, GameObject, Logger} from "./src/index.js";
 import sirv from "sirv";
+import express from "express";
 
 Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow","user", "assistant", "swagger"])
 Config.Set("middlewares", []);
@@ -14,8 +15,8 @@ const bench = GameObject.Create("Benchmark");
 const timer = bench.addComponent(BenchmarkTool);
 timer.start();
 
-
-const engine = await Engine.Create();
+const app = express();
+const engine = await Engine.Create({app});
 
 if (process.argv.length === 3) {
     let arg = process.argv[2];

package/src/constants.js

@@ -64,6 +64,8 @@ export const emailDefaultConfig = {
     pass: 'password'
 }
 
+export const useAI = true;
+
 /**
  * Maximum number of models per user
  * @type {number}
@@ -212,6 +214,7 @@ export const maxTotalPrivateFilesSize = 250 * megabytes;
  */
 export const timeoutVM = 5000;
 
+export const defaultMaxRequestData = 50000;
 /**
  * Options for the HTML sanitizer
  * @type {{allowedSchemesByTag: {}, selfClosing: string[], allowedSchemes: string[], enforceHtmlBoundary: boolean, disallowedTagsMode: string, allowProtocolRelative: boolean, allowedAttributes: {a: string[], img: string[], code: string[]}, allowedTags: string[], allowedSchemesAppliedToAttributes: string[]}}

package/src/engine.js

@@ -26,7 +26,7 @@ export const MongoDatabase = MongoClient.db(dbName);
 
 
 export const Engine = {
-    Create: async (options) => {
+    Create: async (options = { app : null}) => {
         const engine = GameObject.Create("Engine");
         console.log("Creating engine", Config.Get('modules'));
         engine.addComponent(Logger);
@@ -38,8 +38,11 @@ export const Engine = {
             engine.getComponent(Logger).info(`Custom UserProvider '${providerInstance.constructor.name}' has been set.`);
         };
 
+        if (!options.app) {
+            options.app = express();
+        }
 
-        const app = express();
+        const { app }  = options;
         // Allows you to set port in the project properties.
         app.set('port', process.env.PORT || 3000);
         app.set('engine', engine);
@@ -106,12 +109,8 @@ export const Engine = {
             engine._modules = e;
             return Promise.resolve();
         });
-        let server;
 
-        app.use(sirv('client/dist', {
-            single: true,
-            dev: process.env.NODE_ENV === 'development'
-        }));
+        let server;
 
         engine.start = async (port, cb) =>{
             // Use connect method to connect to the server
@@ -132,6 +131,18 @@ export const Engine = {
             if (cb)
                 await cb();
 
+            engine.get('/api/health', (req, res) => {
+                res.status(200).json({
+                    status: 'ok',
+                    timestamp: new Date().toISOString()
+                });
+            });
+
+            app.use(sirv('client/dist', {
+                single: true,
+                dev: process.env.NODE_ENV === 'development'
+            }));
+
             process.on('uncaughtException', function (exception) {
                 console.error(exception);
                 fs.appendFile('issues.txt', JSON.stringify({ code: exception.code, message: exception.message, stack: exception.stack }), function (err) {
@@ -171,12 +182,6 @@ export const Engine = {
         engine.resetModels = async () => {
             await deleteModels();
         };
-        engine.get('/api/health', (req, res) => {
-            res.status(200).json({
-                status: 'ok',
-                timestamp: new Date().toISOString()
-            });
-        });
         return engine;
     }
 }

package/src/i18n.js

@@ -1,11 +1,17 @@
 import i18n from "i18next";
-import {initReactI18next} from "react-i18next";
+import {Trans, useTranslation, initReactI18next} from "react-i18next";
 import LanguageDetector from "i18next-browser-languagedetector";
 
 export const translations = {
     fr: {
         translation: {
+            "datatable.advancedFilter.title": "Filtre avancé (MongoDB)",
+            "datatable.advancedFilter.desc": "Créez de toute pièce vos filtres avancés, avec les opérateurs d'aggrégation de MongoDB disponibles ici.",
 
+            "editData": "Éditer dans {{0}}",
+            "btns.duplicate": "Dupliquer",
+            "btns.edit": "Modifier",
+            "btns.delete": "Supprimer",
             "field_return_status_hint": "Le statut actuel de la demande de retour (ex: Demandé, Approuvé, Remboursé).",
             "field_workflowRun_completedAt_hint": "Horodatage de la fin de l'exécution (réussie ou échouée).",
             "field_employee_socialSecurityNumber_hint": "Numéro de sécurité sociale de l'employé.",
@@ -1235,6 +1241,13 @@ export const translations = {
     en: {
         translation: {
 
+            "datatable.advancedFilter.title": "Advanced Filter (MongoDB)",
+            "datatable.advancedFilter.desc": "Create your own advanced filters from scratch, using MongoDB's aggregation operators available here.",
+
+            "editData": "Edit in {{0}}",
+            "btns.duplicate": "Duplicate",
+            "btns.edit": "Edit",
+            "btns.delete": "Delete",
             "field_workflowRun_completedAt_hint": "Timestamp of when the execution ended (successfully or failed).",
             "field_employee_socialSecurityNumber_hint": "Employee's social security number.",
 
@@ -2462,6 +2475,13 @@ export const translations = {
     },
     es: {
         translation: {
+            "datatable.advancedFilter.title": "Filtro avanzado (MongoDB)",
+            "datatable.advancedFilter.desc": "Cree sus propios filtros avanzados desde cero, utilizando los operadores de agregación de MongoDB disponibles aquí.",
+
+            "editData": "Editar en {{0}}",
+            "btns.duplicate": "Duplicar",
+            "btns.edit": "Editar",
+            "btns.delete": "Eliminar",
             "field_return_status_hint": "El estado actual de la solicitud de devolución (ej: Solicitado, Aprobado, Reembolsado).",
             "field_workflowRun_completedAt_hint": "Marca de tiempo del final de la ejecución (exitosa o fallida).",
             "field_employee_socialSecurityNumber_hint": "Número de seguridad social del empleado.",
@@ -3724,6 +3744,13 @@ export const translations = {
     },
     pt: {
         translation: {
+            "datatable.advancedFilter.title": "Filtro Avançado (MongoDB)",
+            "datatable.advancedFilter.desc": "Crie os seus próprios filtros avançados a partir do zero, utilizando os operadores de agregação do MongoDB disponíveis aqui.",
+
+            "editData": "Editar em {{0}}",
+            "btns.duplicate": "Duplicar",
+            "btns.edit": "Editar",
+            "btns.delete": "Apagar",
             "assistant.type": "Digite sua mensagem...",
             "assistant.welcome": "Olá! Como posso ajudá-lo com seus dados?",
             "assistant.named": "Assistente {{named}}",
@@ -3789,6 +3816,12 @@ export const translations = {
     },
     de: {
         translation: {
+            "datatable.advancedFilter.title": "Erweiterter Filter (MongoDB)",
+            "datatable.advancedFilter.desc": "Erstellen Sie Ihre eigenen erweiterten Filter von Grund auf mit den hier verfügbaren Aggregationsoperatoren von MongoDB.",
+            "editData": "Bearbeiten in {{0}}",
+            "btns.duplicate": "Duplizieren",
+            "btns.edit": "Bearbeiten",
+            "btns.delete": "Löschen",
             "field_workflowRun_completedAt_hint": "Zeitstempel des Abschlusses (erfolgreich oder fehlgeschlagen).",
             "field_employee_socialSecurityNumber_hint": "Sozialversicherungsnummer des Mitarbeiters.",
             "field_shipment_order_hint": "Die mit dieser Sendung verknüpfte Bestellung.",
@@ -4931,6 +4964,12 @@ export const translations = {
     },
     it: {
         translation: {
+            "datatable.advancedFilter.title": "Filtro avanzato (MongoDB)",
+            "datatable.advancedFilter.desc": "Crea i tuoi filtri avanzati da zero, utilizzando gli operatori di aggregazione di MongoDB disponibili qui.",
+            "editData": "Modifica in {{0}}",
+            "btns.duplicate": "Duplica",
+            "btns.edit": "Modifica",
+            "btns.delete": "Elimina",
             "field_workflowRun_completedAt_hint": "Timestamp del termine dell'esecuzione (successo o fallimento).",
             "field_employee_socialSecurityNumber_hint": "Numero di previdenza sociale del dipendente.",
             "field_shipment_order_hint": "L'ordine associato a questa spedizione.",
@@ -6126,6 +6165,12 @@ export const translations = {
     },
     cs: {
         translation: {
+            "datatable.advancedFilter.title": "Rozšířený filtr (MongoDB)",
+            "datatable.advancedFilter.desc": "Vytvořte si vlastní pokročilé filtry od nuly pomocí agregačních operátorů MongoDB dostupných zde.",
+            "editData": "Upravit v {{0}}",
+            "btns.duplicate": "Duplikovat",
+            "btns.edit": "Upravit",
+            "btns.delete": "Smazat",
             "field_return_status_hint": "Aktuální stav žádosti o vrácení (např.: Požadováno, Schváleno, Vráceno).",
             "field_workflowRun_completedAt_hint": "Časové razítko dokončení provedení (úspěšného nebo neúspěšného).",
             "field_employee_socialSecurityNumber_hint": "Rodné číslo zaměstnance.",
@@ -6966,6 +7011,12 @@ export const translations = {
     },
     ru: {
         translation: {
+            "datatable.advancedFilter.title": "Расширенный фильтр (MongoDB)",
+            "datatable.advancedFilter.desc": "Создайте свои собственные расширенные фильтры с нуля, используя операторы агрегации MongoDB, доступные здесь",
+            "editData": "Редактировать в {{0}}",
+            "btns.duplicate": "Дублировать",
+            "btns.edit": "Редактировать",
+            "btns.delete": "Удалить",
             "field_workflowRun_completedAt_hint": "Метка времени завершения выполнения (успешного или неудачного).",
             "field_employee_socialSecurityNumber_hint": "Номер социального страхования сотрудника.",
             "field_shipment_order_hint": "Заказ, связанный с этой отгрузкой.",
@@ -8161,7 +8212,12 @@ export const translations = {
     },
     ar: {
         translation: {
-
+            "datatable.advancedFilter.title": "مرشح متقدم (MongoDB)",
+            "datatable.advancedFilter.desc": "أنشئ مرشحاتك المتقدمة من الصفر، باستخدام عوامل تجميع MongoDB المتوفرة هنا",
+            "editData": "تعديل في {{0}}",
+            "btns.duplicate": "تكرار",
+            "btns.edit": "تعديل",
+            "btns.delete": "حذف",
             "field_return_status_hint": "الحالة الحالية لطلب الإرجاع (مثال: مطلوب، معتمد، تم استرداده).",
             "field_workflowRun_completedAt_hint": "طابع زمني لانتهاء التنفيذ (نجاحًا أو فشلًا).",
             "field_employee_socialSecurityNumber_hint": "رقم الضمان الاجتماعي للموظف.",
@@ -9407,7 +9463,12 @@ export const translations = {
     },
     sv: {
         translation: {
-
+            "datatable.advancedFilter.title": "Avancerat filter (MongoDB)",
+            "datatable.advancedFilter.desc": "Skapa dina egna avancerade filter från grunden med hjälp av MongoDBs aggregeringsoperatorer som finns här.",
+            "editData": "Redigera i {{0}}",
+            "btns.duplicate": "Duplicera",
+            "btns.edit": "Redigera",
+            "btns.delete": "Ta bort",
             "field_return_status_hint": "Den aktuella statusen för returförfrågan (t.ex. Begärd, Godkänd, Återbetald).",
             "field_workflowRun_completedAt_hint": "Tidsstämpel för när körningen avslutades (lyckad eller misslyckad).",
             "field_employee_socialSecurityNumber_hint": "Anställds personnummer.",
@@ -10551,6 +10612,12 @@ export const translations = {
     },
     el: {
         translation: {
+            "datatable.advancedFilter.title": "Προηγμένο Φίλτρο (MongoDB)",
+            "datatable.advancedFilter.desc": "Δημιουργήστε τα δικά σας προηγμένα φίλτρα από την αρχή, χρησιμοποιώντας τους τελεστές συνάθροισης της MongoDB που είναι διαθέσιμοι εδώ.",
+            "editData": "Επεξεργασία στο {{0}}",
+            "btns.duplicate": "Διπλότυπο",
+            "btns.edit": "Επεξεργασία",
+            "btns.delete": "Διαγραφή",
             "field_workflowRun_completedAt_hint": "Χρονική σήμανση της ολοκλήρωσης της εκτέλεσης (επιτυχής ή αποτυχημένη).",
             "field_employee_socialSecurityNumber_hint": "Ο αριθμός κοινωνικής ασφάλισης του υπαλλήλου.",
             "field_shipment_order_hint": "Η παραγγελία που σχετίζεται με αυτήν την αποστολή.",
@@ -11680,6 +11747,12 @@ export const translations = {
     },
     fa: {
         translation: {
+            "datatable.advancedFilter.title": "فیلتر پیشرفته (MongoDB)",
+            "datatable.advancedFilter.desc": "با استفاده از عملگرهای تجمیع MongoDB که در اینجا موجود است، فیلترهای پیشرفته خود را از ابتدا ایجاد کنید.",
+            "editData": "ویرایش در {{0}}",
+            "btns.duplicate": "کپی",
+            "btns.edit": "ویرایش",
+            "btns.delete": "حذف",
             "field_return_status_hint": "وضعیت فعلی درخواست مرجوعی (مثال: درخواست شده، تأیید شده، مسترد شده).",
             "field_workflowRun_completedAt_hint": "مهر زمانی پایان اجرا (موفق یا ناموفق).",
             "field_employee_socialSecurityNumber_hint": "شماره تأمین اجتماعی کارمند.",
@@ -12368,4 +12441,5 @@ i18n
         }
     });
 
+export {Trans, useTranslation};
 export default i18n;

package/src/modules/assistant.js

@@ -7,7 +7,7 @@ import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
 import { HumanMessage, SystemMessage } from "@langchain/core/messages";
 import {searchData,  patchData, deleteData, insertData} from "./data.js";
 import { getDataAsString } from "../data.js";
-import i18n from "data-primals-engine/i18n";
+import i18n from "../../src/i18n.js";
 import {generateLimiter} from "./user.js";
 import rateLimit from "express-rate-limit";
 

package/src/modules/bucket.js

@@ -9,7 +9,7 @@ import {Logger} from "../gameObject.js";
 import {middlewareAuthenticator, userInitiator} from "./user.js";
 import {awsDefaultConfig, maxBytesPerSecondThrottleData} from "../constants.js";
 import crypto from "node:crypto";
-import i18n from "data-primals-engine/i18n";
+import i18n from "../../src/i18n.js";
 import {sendEmail} from "../email.js";
 import {throttleMiddleware} from "../middlewares/throttle.js";
 

package/src/modules/data.js

@@ -64,7 +64,7 @@ import {
 import fs from "node:fs";
 import schedule from "node-schedule";
 import {middleware} from "../middlewares/middleware-mongodb.js";
-import i18n from "data-primals-engine/i18n";
+import i18n from "../i18n.js";
 import {
     executeSafeJavascript,
     runScheduledJobWithDbLock,
@@ -4038,9 +4038,18 @@ export const searchData = async ({user, query}) => {
                                                 {$eq: ["$_model", fi.relation]},
                                                 {$eq: ["$_user", user.username]},
                                                 fi.multiple ? {
-                                                    $in: [{$toString: "$_id"}, {$ifNull: ['$$convertedId', []]}]
+                                                    $in: [{ $toString: "$_id" }, {
+                                                        $map: {
+                                                            input: { $ifNull: ["$$convertedId", []] }, // On utilise le tableau d'IDs, ou un tableau vide s'il est null
+                                                            as: "relationId",
+                                                            in: { $toString: "$$relationId" }
+                                                        }
+                                                    }]
                                                 } : {
-                                                    $eq: [{$toString: "$_id"}, '$$convertedId']
+                                                    $eq: [
+                                                        { $toString: "$_id" },
+                                                        { $convert: { input: '$$convertedId', to: "string", onError: '' } }
+                                                    ]
                                                 }
                                             ]
                                         ]
@@ -4315,6 +4324,7 @@ export const searchData = async ({user, query}) => {
             } else if (fi.type === 'array') {
                 // Handle array filtering here
                 if (data[fi.name]) {
+                    console.log('array filtering', data[fi.name]);
                     pipelines.push({
                         $match: {
                             $expr: {
@@ -4350,6 +4360,7 @@ export const searchData = async ({user, query}) => {
 
     let pipelines = [];
     if( allIds.length ){
+        console.log({allIds});
         const id = {$in: ["$_id", allIds.map(m => new ObjectId(m))]};
         pipelines.push({
             $match: { $expr: id }

package/src/modules/file.js

@@ -2,7 +2,7 @@
 // Nouvelle fonction pour ajouter un fichier privé
 import {maxPrivateFileSize, megabytes} from "../constants.js";
 import {isLocalUser} from "../data.js";
-import i18n from "data-primals-engine/i18n";
+import i18n from "../../src/i18n.js";
 import {getCollection} from "./mongodb.js";
 import {getFileExtension, isGUID, uuidv4} from "../core.js";
 import path from "node:path";

package/src/modules/user.js

@@ -1,4 +1,4 @@
-import i18n from "data-primals-engine/i18n";
+import i18n from "../i18n.js";
 import {MongoDatabase} from "../engine.js";
 import {getCollection, getCollectionForUser, getUserCollectionName} from "./mongodb.js";
 import {isLocalUser} from "../data.js";

package/src/modules/workflow.js

@@ -12,7 +12,7 @@ import {maxExecutionsByStep, maxWorkflowSteps, timeoutVM} from "../constants.js"
 import {ChatOpenAI} from "@langchain/openai";
 import {ChatGoogleGenerativeAI} from "@langchain/google-genai";
 import {ChatPromptTemplate} from "@langchain/core/prompts";
-import i18n from "data-primals-engine/i18n";
+import i18n from "../../src/i18n.js";
 import {sendEmail} from "../email.js";
 
 import * as workflowModule from './workflow.js';

package/src/packs.js

@@ -390,8 +390,9 @@ export const getAllPacks = async () => {
                         return {
                             name: m.name,
                             permissions: {
-                                "$find": {
-                                    "$in": ["$name", m.perms]
+                                "$link": {
+                                    "$in": ["$name", m.perms],
+                                    "_model": "permission"
                                 }
                             }
                         }
@@ -407,7 +408,7 @@ export const getAllPacks = async () => {
                         "code": "fr"
                     }],
                     "translation": [
-                        { "lang": { "$find": { "$eq": ["$code", "fr"]}}, "key": "Visitor alerts", "value": "Alertes visiteurs" }
+                        { "lang": { "$link": { "$eq": ["$code", "fr"]}, "_model": "lang"}, "key": "Visitor alerts", "value": "Alertes visiteurs" }
                     ]
                 }
             }

package/src/setenv.js

@@ -1,4 +1,4 @@
-import {getRandom} from "data-primals-engine/core";
+import {getRandom} from "./core.js";
 import process from "node:process";
 import {Engine} from "./index.js";
 

package/test/data.integration.test.js

@@ -1,14 +1,13 @@
 // __tests__/data.integration.test.js
 import { ObjectId } from 'mongodb';
-import {expect, describe, it, beforeEach, afterEach, beforeAll, afterAll, vi} from 'vitest';
+import {expect, describe, it, beforeEach, beforeAll} from 'vitest';
 import { Config } from '../src/config.js';
 
-// --- Importations des modules de votre application ---
 import {
     insertData,
     editData,
     deleteData,
-    getModel, searchData, installPack
+    searchData, installPack
 } from 'data-primals-engine/modules/data';
 
 import {
@@ -16,13 +15,8 @@ import {
     getCollection,
     getCollectionForUser as getAppUserCollection
 } from 'data-primals-engine/modules/mongodb';
-import { Engine } from "data-primals-engine/engine";
-import process from "node:process";
-import fs from "node:fs";
-import util from "node:util";
-import {getRandom} from "data-primals-engine/core";
-import {generateUniqueName, getUniquePort, initEngine} from "../src/setenv.js";
-import {MongoDatabase} from "../src/engine.js";
+import {getRandom} from "../src/core.js";
+import {generateUniqueName, initEngine} from "../src/setenv.js";
 
 let testModelsColInstance;
 let testDatasColInstance;

package/test/workflow.robustness.test.js

@@ -6,7 +6,7 @@ import { insertData, editData } from 'data-primals-engine/modules/data';
 import { modelsCollection as getAppModelsCollection, getCollectionForUser, getCollection } from 'data-primals-engine/modules/mongodb';
 import * as workflowModule from 'data-primals-engine/modules/workflow';
 import {initEngine} from "../src/setenv.js";
-import {maxExecutionsByStep} from "data-primals-engine/constants";
+import {maxExecutionsByStep} from "../src/constants.js";
 
 vi.mock('data-primals-engine/modules/workflow', { spy: true })