data-primals-engine 1.4.3 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +878 -867
  2. package/client/package-lock.json +49 -0
  3. package/client/package.json +1 -0
  4. package/client/src/App.jsx +1 -1
  5. package/client/src/App.scss +13 -3
  6. package/client/src/AssistantChat.scss +3 -2
  7. package/client/src/DashboardView.jsx +569 -569
  8. package/client/src/DataEditor.jsx +2 -2
  9. package/client/src/DataLayout.jsx +1 -8
  10. package/client/src/DataTable.jsx +26 -4
  11. package/client/src/Field.jsx +1825 -1788
  12. package/client/src/FlexDataRenderer.jsx +2 -0
  13. package/client/src/FlexTreeUtils.js +1 -1
  14. package/client/src/KPIDialog.jsx +11 -1
  15. package/client/src/ModelCreator.jsx +1 -2
  16. package/client/src/ModelCreatorField.jsx +23 -27
  17. package/client/src/ModelList.jsx +1 -1
  18. package/client/src/constants.js +1 -1
  19. package/client/src/hooks/useTutorials.jsx +62 -65
  20. package/client/src/translations.js +2 -0
  21. package/package.json +2 -1
  22. package/perf/README.md +147 -0
  23. package/perf/artillery-hooks.js +37 -0
  24. package/perf/perf-shot-hardwork.yml +84 -0
  25. package/perf/perf-shot-search.yml +45 -0
  26. package/perf/setup.yml +26 -0
  27. package/server.js +1 -1
  28. package/src/constants.js +2 -27
  29. package/src/core.js +15 -1
  30. package/src/data.js +1 -1
  31. package/src/defaultModels.js +1544 -1540
  32. package/src/email.js +5 -2
  33. package/src/engine.js +5 -3
  34. package/src/filter.js +5 -3
  35. package/src/modules/assistant/assistant.js +3 -1
  36. package/src/modules/bucket.js +12 -15
  37. package/src/modules/data/data.backup.js +11 -8
  38. package/src/modules/data/data.js +6 -3
  39. package/src/modules/data/data.operations.js +3231 -2999
  40. package/src/modules/data/data.routes.js +1821 -1785
  41. package/src/modules/data/data.scheduling.js +2 -1
  42. package/src/modules/data/data.validation.js +4 -1
  43. package/src/modules/file.js +4 -2
  44. package/src/modules/user.js +4 -1
  45. package/src/modules/workflow.js +9 -10
  46. package/src/openai.jobs.js +2 -0
  47. package/src/packs.js +22 -5
  48. package/src/providers.js +22 -7
  49. package/test/data.integration.test.js +1060 -981
  50. package/test/import_export.integration.test.js +1 -1
package/src/email.js CHANGED
@@ -3,6 +3,7 @@ import nodemailer from "nodemailer";
3
3
  import juice from "juice";
4
4
  import {Event} from "./events.js";
5
5
  import {emailDefaultConfig} from "./constants.js";
6
+ import {Config} from "./config.js";
6
7
 
7
8
  // Le transporteur par défaut, utilisé si aucune config spécifique n'est fournie.
8
9
  const defaultTransporter = nodemailer.createTransport({
@@ -45,12 +46,14 @@ const createTransporter = (smtpConfig) => {
45
46
  * @param {string|null} tpl - Le template HTML à utiliser.
46
47
  */
47
48
  export const sendEmail = async (email = "", data, smtpConfig = null, lang, tpl = null) => {
48
- const contactEmail = smtpConfig ? (smtpConfig.from || emailDefaultConfig.from) :"Our company <noreply@ourdomain.tld>";
49
+
50
+ const cfg = Config.Get('emailDefaultConfig', emailDefaultConfig);
51
+ const contactEmail = smtpConfig ? (smtpConfig.from || cfg.from) :"Our company <noreply@ourdomain.tld>";
49
52
  const emails = Array.isArray(email) ? email : [email];
50
53
  if (emails.length === 0) return false;
51
54
 
52
55
  // Choisir le transporteur à utiliser
53
- const transporter = smtpConfig ? createTransporter(smtpConfig||emailDefaultConfig) : defaultTransporter;
56
+ const transporter = smtpConfig ? createTransporter(smtpConfig||cfg) : defaultTransporter;
54
57
 
55
58
  Event.Listen("OnEmailTemplate", (data, lang) => data.content, "event", "system");
56
59
 
package/src/engine.js CHANGED
@@ -7,7 +7,7 @@ import process from "process";
7
7
  import {
8
8
  cookiesSecret,
9
9
  databasePoolSize,
10
- dbName,
10
+ dbName as dbNameBase,
11
11
  tlsAllowInvalidCertificates, tlsAllowInvalidHostnames
12
12
  } from "./constants.js";
13
13
  import http from "http";
@@ -32,7 +32,7 @@ import {createModel, deleteModels, getModels, installAllPacks} from "./modules/d
32
32
  const __filename = fileURLToPath(import.meta.url);
33
33
  const __dirname = path.dirname(__filename);
34
34
 
35
-
35
+ let dbName = Config.Get('dbName', dbNameBase);
36
36
  let caFile, certFile, keyFile;
37
37
  try {
38
38
  if (process.env.CA_CERT)
@@ -127,7 +127,9 @@ export const Engine = {
127
127
  uploadDir: process.cwd()+'/uploads/tmp',
128
128
  multiples: true // req.files to be arrays of files
129
129
  }));
130
- app.use(cookieParser(process.env.COOKIES_SECRET || cookiesSecret));
130
+
131
+ const cs = Config.Get('cookieSecret', process.env.COOKIES_SECRET || cookiesSecret)
132
+ app.use(cookieParser(cs));
131
133
  app.use(requestIp.mw())
132
134
 
133
135
  engine.use = (...args) => {
package/src/filter.js CHANGED
@@ -36,12 +36,14 @@ const evaluateSingleCondition = (currentModelDef, condition, formData, allModels
36
36
 
37
37
  // Gestion des opérateurs d'agrégation (commencent par $)
38
38
  const operator = Object.keys(condition)[0];
39
- if (operator === '$eq' && Array.isArray(condition[operator])) {
40
- // Cas spécial pour {$eq: ['$field', value]}
39
+ if ((operator === '$eq' || operator === '$ne') && Array.isArray(condition[operator])) {
40
+ // Cas spécial pour {$op: ['$field', value]}
41
41
  const [fieldPath, expectedValue] = condition[operator];
42
42
  if (typeof fieldPath === 'string' && fieldPath.startsWith('$')) {
43
43
  const fieldName = fieldPath.substring(1); // Enlève le $ devant
44
- return formData[fieldName] == expectedValue;
44
+ const actualValue = formData[fieldName];
45
+ if (operator === '$eq') return actualValue == expectedValue;
46
+ if (operator === '$ne') return actualValue != expectedValue;
45
47
  }
46
48
  }
47
49
 
@@ -12,6 +12,7 @@ import {maxAIReflectiveSteps} from "../../constants.js";
12
12
  import {providers} from "./constants.js";
13
13
  import {ChatDeepSeek} from "@langchain/deepseek";
14
14
  import {ChatAnthropic} from "@langchain/anthropic";
15
+ import {Config} from "../../config.js";
15
16
 
16
17
  let logger = null;
17
18
 
@@ -356,7 +357,8 @@ async function handleChatRequest(message, history, provider, context, user, conf
356
357
 
357
358
 
358
359
  // --- BOUCLE DE RAISONNEMENT ET D'EXÉCUTION D'OUTILS ---
359
- for (let i = 0; i < maxAIReflectiveSteps; i++) {
360
+ const m = Config.Get('maxAIReflectiveSteps', maxAIReflectiveSteps);
361
+ for (let i = 0; i < m; i++) {
360
362
  logger.debug(`[Assistant] Tour de boucle ${i + 1}. Invocation de l'IA...`);
361
363
 
362
364
  const response = await llm.invoke(conversationHistory);
@@ -13,6 +13,7 @@ import {sendEmail} from "../email.js";
13
13
  import {throttleMiddleware} from "../middlewares/throttle.js";
14
14
  import {getCollectionForUser} from "./mongodb.js";
15
15
  import {safeAssignObject} from "../core.js";
16
+ import {Config} from "../config.js";
16
17
 
17
18
  const restoreRequests = {};
18
19
 
@@ -47,15 +48,6 @@ export const requestRestore = async (user, lang) => {
47
48
  }
48
49
  };
49
50
 
50
- const getDefaultS3Config = () => {
51
- return {
52
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
53
- secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, // Utiliser la clé déchiffrée
54
- region: process.env.AWS_REGION || awsDefaultConfig.region,
55
- bucketName: process.env.AWS_BUCKET || awsDefaultConfig.bucketName
56
- };
57
- }
58
-
59
51
  /**
60
52
  * Récupère la configuration S3 depuis les variables d'environnement de l'utilisateur en base de données.
61
53
  * @param {object} user - L'objet utilisateur.
@@ -98,12 +90,16 @@ async function _getUserS3ConfigFromDb(user) {
98
90
  * @returns {Promise<object>} - L'objet de configuration S3 final.
99
91
  */
100
92
  export async function getUserS3Config(user) {
93
+
94
+ const adc = awsDefaultConfig;
95
+ const dc = Config.Get('awsDefaultConfig', {});
96
+
101
97
  // 1. Récupérer la configuration globale par défaut
102
98
  const defaultConfig = {
103
- accessKeyId: process.env.AWS_ACCESS_KEY_ID,
104
- secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
105
- region: process.env.AWS_REGION || awsDefaultConfig.region,
106
- bucketName: process.env.AWS_BUCKET_NAME || awsDefaultConfig.bucketName
99
+ accessKeyId: dc.accessKeyId || process.env.AWS_ACCESS_KEY_ID,
100
+ secretAccessKey: dc.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY,
101
+ region: dc.region || process.env.AWS_REGION || adc.region,
102
+ bucketName: dc.bucketName || process.env.AWS_BUCKET_NAME || adc.bucketName
107
103
  };
108
104
 
109
105
  // 2. Récupérer la configuration spécifique de l'utilisateur
@@ -243,13 +239,14 @@ export const getS3Stream = (s3Config, s3Key) => {
243
239
  return s3.getObject(params).createReadStream();
244
240
  };
245
241
 
246
- const throttle = throttleMiddleware(maxBytesPerSecondThrottleData);
247
-
248
242
  let engine, logger;
249
243
  export async function onInit(defaultEngine) {
250
244
  engine = defaultEngine;
251
245
  logger = engine.getComponent(Logger);
252
246
 
247
+ const m = Config.Get('maxBytesPerSecondThrottleData', maxBytesPerSecondThrottleData);
248
+ const throttle = throttleMiddleware(m);
249
+
253
250
  const userMiddlewares = await engine.userProvider.getMiddlewares();
254
251
  engine.post('/api/backup/request-restore', [throttle, middlewareAuthenticator, userInitiator, ...userMiddlewares], async (req, res) => {
255
252
  const user = req.me; // Assuming you have user authentication middleware
@@ -153,7 +153,8 @@ export const loadFromDump = async (user, options = {}) => {
153
153
  }
154
154
 
155
155
  // --- EXÉCUTION DE MONGORESTORE ---
156
- const restoreSourceDir = path.join(tmpRestoreDir, dbName);
156
+ const d = Config.Get('dbName', dbName);
157
+ const restoreSourceDir = path.join(tmpRestoreDir, d);
157
158
  if (!fs.existsSync(restoreSourceDir)) {
158
159
  throw new Error(`Restore source directory (${restoreSourceDir}) not found.`);
159
160
  }
@@ -161,15 +162,15 @@ export const loadFromDump = async (user, options = {}) => {
161
162
  let command;
162
163
  const args = [
163
164
  '--uri', dbUrl,
164
- '--db', dbName
165
+ '--db', d
165
166
  ];
166
167
 
167
168
  if (modelsOnly) {
168
- args.push('--nsInclude', `${dbName}.models`);
169
+ args.push('--nsInclude', `${d}.models`);
169
170
  } else {
170
171
  // mongorestore accepte plusieurs fois l'option --nsInclude
171
- args.push('--nsInclude', `${dbName}.datas`);
172
- args.push('--nsInclude', `${dbName}.models`);
172
+ args.push('--nsInclude', `${d}.datas`);
173
+ args.push('--nsInclude', `${d}.models`);
173
174
  }
174
175
  // Le répertoire source est le dernier argument
175
176
  args.push(restoreSourceDir);
@@ -242,6 +243,8 @@ export const dumpUserData = async (user) => {
242
243
  const backupFrequency = await engine.userProvider.getBackupFrequency(user);
243
244
  logger.info(`Fréquence de sauvegarde : ${backupFrequency}.`);
244
245
 
246
+
247
+ const d = Config.Get('dbName', dbName);
245
248
  const collections = await MongoDatabase.listCollections().toArray();
246
249
  for (const collection of collections) {
247
250
  const collsToBackup = [await getUserCollectionName(user), 'models'];
@@ -249,7 +252,7 @@ export const dumpUserData = async (user) => {
249
252
  const query = {_user: user.username};
250
253
  const args = [
251
254
  '--uri', dbUrl,
252
- '--db', dbName,
255
+ '--db', d,
253
256
  '--out', localTempDumpDir,
254
257
  '--collection', collection.name,
255
258
  '--query', JSON.stringify(query)
@@ -258,9 +261,9 @@ export const dumpUserData = async (user) => {
258
261
  await execFileAsync('mongodump', args);
259
262
  }
260
263
  }
261
- const dumpSourceDir = path.join(localTempDumpDir, dbName);
264
+ const dumpSourceDir = path.join(localTempDumpDir, d);
262
265
  if (fs.existsSync(dumpSourceDir)) {
263
- await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [dbName]);
266
+ await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [d]);
264
267
  logger.info(`Archive de sauvegarde locale créée : ${localArchivePath}`);
265
268
  } else {
266
269
  logger.warn(`Le répertoire de dump ${dumpSourceDir} était vide. Aucune archive n'a créée.`);
@@ -2,7 +2,7 @@ import {Logger} from "../../gameObject.js";
2
2
  import {mkdir} from 'node:fs/promises';
3
3
  import {onInit as historyInit} from "./data.history.js";
4
4
  import {isDemoUser, isLocalUser} from "../../data.js";
5
- import {install, maxRequestData, storageSafetyMargin} from "../../constants.js";
5
+ import {install, maxAlertsPerUser, maxRequestData, storageSafetyMargin} from "../../constants.js";
6
6
  import {createCollection, getCollection} from "../mongodb.js";
7
7
  import path from "node:path";
8
8
  import {isGUID, sequential} from "../../core.js";
@@ -22,6 +22,7 @@ import {validateField, onInit as validationInit} from "./data.validation.js";
22
22
  import {cancelAlerts, scheduleAlerts, onInit as scheduleInit} from "./data.scheduling.js";
23
23
  import {deleteData, installPack, onInit as operationsInit} from "./data.operations.js";
24
24
  import {jobDumpUserData, onInit as backupInit} from "./data.backup.js";
25
+ import {Config} from "../../config.js";
25
26
 
26
27
  let engine;
27
28
  let logger;
@@ -54,7 +55,8 @@ export async function onInit(defaultEngine) {
54
55
 
55
56
  let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection, historyCollection;
56
57
 
57
- if( install ) {
58
+ const i = Config.Get('install', install);
59
+ if( i ) {
58
60
  datasCollection = await createCollection("datas");
59
61
  historyCollection = await createCollection("history");
60
62
  filesCollection = await createCollection("files");
@@ -220,8 +222,9 @@ export async function checkServerCapacity(incomingDataSize = 0) {
220
222
  const diskSpace = await checkDiskSpace(DATA_STORAGE_PATH);
221
223
  const { free, size } = diskSpace;
222
224
 
225
+ const storageMargin = Config.Get('storageSafetyMargin', storageSafetyMargin);
223
226
  // Limite maximale d'utilisation du disque (ex: 90% de la taille totale)
224
- const maxAllowedUsage = size * storageSafetyMargin;
227
+ const maxAllowedUsage = size * storageMargin;
225
228
  const currentUsage = size - free;
226
229
  const projectedUsage = currentUsage + incomingDataSize;
227
230