data-primals-engine 1.4.3 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +878 -867
- package/client/package-lock.json +49 -0
- package/client/package.json +1 -0
- package/client/src/App.jsx +1 -1
- package/client/src/App.scss +13 -3
- package/client/src/AssistantChat.scss +3 -2
- package/client/src/DashboardView.jsx +569 -569
- package/client/src/DataEditor.jsx +2 -2
- package/client/src/DataLayout.jsx +1 -8
- package/client/src/DataTable.jsx +26 -4
- package/client/src/Field.jsx +1825 -1788
- package/client/src/FlexDataRenderer.jsx +2 -0
- package/client/src/FlexTreeUtils.js +1 -1
- package/client/src/KPIDialog.jsx +11 -1
- package/client/src/ModelCreator.jsx +1 -2
- package/client/src/ModelCreatorField.jsx +23 -27
- package/client/src/ModelList.jsx +1 -1
- package/client/src/constants.js +1 -1
- package/client/src/hooks/useTutorials.jsx +62 -65
- package/client/src/translations.js +2 -0
- package/package.json +2 -1
- package/perf/README.md +147 -0
- package/perf/artillery-hooks.js +37 -0
- package/perf/perf-shot-hardwork.yml +84 -0
- package/perf/perf-shot-search.yml +45 -0
- package/perf/setup.yml +26 -0
- package/server.js +1 -1
- package/src/constants.js +2 -27
- package/src/core.js +15 -1
- package/src/data.js +1 -1
- package/src/defaultModels.js +1544 -1540
- package/src/email.js +5 -2
- package/src/engine.js +5 -3
- package/src/filter.js +5 -3
- package/src/modules/assistant/assistant.js +3 -1
- package/src/modules/bucket.js +12 -15
- package/src/modules/data/data.backup.js +11 -8
- package/src/modules/data/data.js +6 -3
- package/src/modules/data/data.operations.js +3231 -2999
- package/src/modules/data/data.routes.js +1821 -1785
- package/src/modules/data/data.scheduling.js +2 -1
- package/src/modules/data/data.validation.js +4 -1
- package/src/modules/file.js +4 -2
- package/src/modules/user.js +4 -1
- package/src/modules/workflow.js +9 -10
- package/src/openai.jobs.js +2 -0
- package/src/packs.js +22 -5
- package/src/providers.js +22 -7
- package/test/data.integration.test.js +1060 -981
- package/test/import_export.integration.test.js +1 -1
package/src/email.js
CHANGED
|
@@ -3,6 +3,7 @@ import nodemailer from "nodemailer";
|
|
|
3
3
|
import juice from "juice";
|
|
4
4
|
import {Event} from "./events.js";
|
|
5
5
|
import {emailDefaultConfig} from "./constants.js";
|
|
6
|
+
import {Config} from "./config.js";
|
|
6
7
|
|
|
7
8
|
// Le transporteur par défaut, utilisé si aucune config spécifique n'est fournie.
|
|
8
9
|
const defaultTransporter = nodemailer.createTransport({
|
|
@@ -45,12 +46,14 @@ const createTransporter = (smtpConfig) => {
|
|
|
45
46
|
* @param {string|null} tpl - Le template HTML à utiliser.
|
|
46
47
|
*/
|
|
47
48
|
export const sendEmail = async (email = "", data, smtpConfig = null, lang, tpl = null) => {
|
|
48
|
-
|
|
49
|
+
|
|
50
|
+
const cfg = Config.Get('emailDefaultConfig', emailDefaultConfig);
|
|
51
|
+
const contactEmail = smtpConfig ? (smtpConfig.from || cfg.from) :"Our company <noreply@ourdomain.tld>";
|
|
49
52
|
const emails = Array.isArray(email) ? email : [email];
|
|
50
53
|
if (emails.length === 0) return false;
|
|
51
54
|
|
|
52
55
|
// Choisir le transporteur à utiliser
|
|
53
|
-
const transporter = smtpConfig ? createTransporter(smtpConfig||
|
|
56
|
+
const transporter = smtpConfig ? createTransporter(smtpConfig||cfg) : defaultTransporter;
|
|
54
57
|
|
|
55
58
|
Event.Listen("OnEmailTemplate", (data, lang) => data.content, "event", "system");
|
|
56
59
|
|
package/src/engine.js
CHANGED
|
@@ -7,7 +7,7 @@ import process from "process";
|
|
|
7
7
|
import {
|
|
8
8
|
cookiesSecret,
|
|
9
9
|
databasePoolSize,
|
|
10
|
-
dbName,
|
|
10
|
+
dbName as dbNameBase,
|
|
11
11
|
tlsAllowInvalidCertificates, tlsAllowInvalidHostnames
|
|
12
12
|
} from "./constants.js";
|
|
13
13
|
import http from "http";
|
|
@@ -32,7 +32,7 @@ import {createModel, deleteModels, getModels, installAllPacks} from "./modules/d
|
|
|
32
32
|
const __filename = fileURLToPath(import.meta.url);
|
|
33
33
|
const __dirname = path.dirname(__filename);
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
let dbName = Config.Get('dbName', dbNameBase);
|
|
36
36
|
let caFile, certFile, keyFile;
|
|
37
37
|
try {
|
|
38
38
|
if (process.env.CA_CERT)
|
|
@@ -127,7 +127,9 @@ export const Engine = {
|
|
|
127
127
|
uploadDir: process.cwd()+'/uploads/tmp',
|
|
128
128
|
multiples: true // req.files to be arrays of files
|
|
129
129
|
}));
|
|
130
|
-
|
|
130
|
+
|
|
131
|
+
const cs = Config.Get('cookieSecret', process.env.COOKIES_SECRET || cookiesSecret)
|
|
132
|
+
app.use(cookieParser(cs));
|
|
131
133
|
app.use(requestIp.mw())
|
|
132
134
|
|
|
133
135
|
engine.use = (...args) => {
|
package/src/filter.js
CHANGED
|
@@ -36,12 +36,14 @@ const evaluateSingleCondition = (currentModelDef, condition, formData, allModels
|
|
|
36
36
|
|
|
37
37
|
// Gestion des opérateurs d'agrégation (commencent par $)
|
|
38
38
|
const operator = Object.keys(condition)[0];
|
|
39
|
-
if (operator === '$eq' && Array.isArray(condition[operator])) {
|
|
40
|
-
// Cas spécial pour {$
|
|
39
|
+
if ((operator === '$eq' || operator === '$ne') && Array.isArray(condition[operator])) {
|
|
40
|
+
// Cas spécial pour {$op: ['$field', value]}
|
|
41
41
|
const [fieldPath, expectedValue] = condition[operator];
|
|
42
42
|
if (typeof fieldPath === 'string' && fieldPath.startsWith('$')) {
|
|
43
43
|
const fieldName = fieldPath.substring(1); // Enlève le $ devant
|
|
44
|
-
|
|
44
|
+
const actualValue = formData[fieldName];
|
|
45
|
+
if (operator === '$eq') return actualValue == expectedValue;
|
|
46
|
+
if (operator === '$ne') return actualValue != expectedValue;
|
|
45
47
|
}
|
|
46
48
|
}
|
|
47
49
|
|
|
@@ -12,6 +12,7 @@ import {maxAIReflectiveSteps} from "../../constants.js";
|
|
|
12
12
|
import {providers} from "./constants.js";
|
|
13
13
|
import {ChatDeepSeek} from "@langchain/deepseek";
|
|
14
14
|
import {ChatAnthropic} from "@langchain/anthropic";
|
|
15
|
+
import {Config} from "../../config.js";
|
|
15
16
|
|
|
16
17
|
let logger = null;
|
|
17
18
|
|
|
@@ -356,7 +357,8 @@ async function handleChatRequest(message, history, provider, context, user, conf
|
|
|
356
357
|
|
|
357
358
|
|
|
358
359
|
// --- BOUCLE DE RAISONNEMENT ET D'EXÉCUTION D'OUTILS ---
|
|
359
|
-
|
|
360
|
+
const m = Config.Get('maxAIReflectiveSteps', maxAIReflectiveSteps);
|
|
361
|
+
for (let i = 0; i < m; i++) {
|
|
360
362
|
logger.debug(`[Assistant] Tour de boucle ${i + 1}. Invocation de l'IA...`);
|
|
361
363
|
|
|
362
364
|
const response = await llm.invoke(conversationHistory);
|
package/src/modules/bucket.js
CHANGED
|
@@ -13,6 +13,7 @@ import {sendEmail} from "../email.js";
|
|
|
13
13
|
import {throttleMiddleware} from "../middlewares/throttle.js";
|
|
14
14
|
import {getCollectionForUser} from "./mongodb.js";
|
|
15
15
|
import {safeAssignObject} from "../core.js";
|
|
16
|
+
import {Config} from "../config.js";
|
|
16
17
|
|
|
17
18
|
const restoreRequests = {};
|
|
18
19
|
|
|
@@ -47,15 +48,6 @@ export const requestRestore = async (user, lang) => {
|
|
|
47
48
|
}
|
|
48
49
|
};
|
|
49
50
|
|
|
50
|
-
const getDefaultS3Config = () => {
|
|
51
|
-
return {
|
|
52
|
-
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
53
|
-
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, // Utiliser la clé déchiffrée
|
|
54
|
-
region: process.env.AWS_REGION || awsDefaultConfig.region,
|
|
55
|
-
bucketName: process.env.AWS_BUCKET || awsDefaultConfig.bucketName
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
|
|
59
51
|
/**
|
|
60
52
|
* Récupère la configuration S3 depuis les variables d'environnement de l'utilisateur en base de données.
|
|
61
53
|
* @param {object} user - L'objet utilisateur.
|
|
@@ -98,12 +90,16 @@ async function _getUserS3ConfigFromDb(user) {
|
|
|
98
90
|
* @returns {Promise<object>} - L'objet de configuration S3 final.
|
|
99
91
|
*/
|
|
100
92
|
export async function getUserS3Config(user) {
|
|
93
|
+
|
|
94
|
+
const adc = awsDefaultConfig;
|
|
95
|
+
const dc = Config.Get('awsDefaultConfig', {});
|
|
96
|
+
|
|
101
97
|
// 1. Récupérer la configuration globale par défaut
|
|
102
98
|
const defaultConfig = {
|
|
103
|
-
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
104
|
-
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
105
|
-
region: process.env.AWS_REGION ||
|
|
106
|
-
bucketName: process.env.AWS_BUCKET_NAME ||
|
|
99
|
+
accessKeyId: dc.accessKeyId || process.env.AWS_ACCESS_KEY_ID,
|
|
100
|
+
secretAccessKey: dc.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY,
|
|
101
|
+
region: dc.region || process.env.AWS_REGION || adc.region,
|
|
102
|
+
bucketName: dc.bucketName || process.env.AWS_BUCKET_NAME || adc.bucketName
|
|
107
103
|
};
|
|
108
104
|
|
|
109
105
|
// 2. Récupérer la configuration spécifique de l'utilisateur
|
|
@@ -243,13 +239,14 @@ export const getS3Stream = (s3Config, s3Key) => {
|
|
|
243
239
|
return s3.getObject(params).createReadStream();
|
|
244
240
|
};
|
|
245
241
|
|
|
246
|
-
const throttle = throttleMiddleware(maxBytesPerSecondThrottleData);
|
|
247
|
-
|
|
248
242
|
let engine, logger;
|
|
249
243
|
export async function onInit(defaultEngine) {
|
|
250
244
|
engine = defaultEngine;
|
|
251
245
|
logger = engine.getComponent(Logger);
|
|
252
246
|
|
|
247
|
+
const m = Config.Get('maxBytesPerSecondThrottleData', maxBytesPerSecondThrottleData);
|
|
248
|
+
const throttle = throttleMiddleware(m);
|
|
249
|
+
|
|
253
250
|
const userMiddlewares = await engine.userProvider.getMiddlewares();
|
|
254
251
|
engine.post('/api/backup/request-restore', [throttle, middlewareAuthenticator, userInitiator, ...userMiddlewares], async (req, res) => {
|
|
255
252
|
const user = req.me; // Assuming you have user authentication middleware
|
|
@@ -153,7 +153,8 @@ export const loadFromDump = async (user, options = {}) => {
|
|
|
153
153
|
}
|
|
154
154
|
|
|
155
155
|
// --- EXÉCUTION DE MONGORESTORE ---
|
|
156
|
-
const
|
|
156
|
+
const d = Config.Get('dbName', dbName);
|
|
157
|
+
const restoreSourceDir = path.join(tmpRestoreDir, d);
|
|
157
158
|
if (!fs.existsSync(restoreSourceDir)) {
|
|
158
159
|
throw new Error(`Restore source directory (${restoreSourceDir}) not found.`);
|
|
159
160
|
}
|
|
@@ -161,15 +162,15 @@ export const loadFromDump = async (user, options = {}) => {
|
|
|
161
162
|
let command;
|
|
162
163
|
const args = [
|
|
163
164
|
'--uri', dbUrl,
|
|
164
|
-
'--db',
|
|
165
|
+
'--db', d
|
|
165
166
|
];
|
|
166
167
|
|
|
167
168
|
if (modelsOnly) {
|
|
168
|
-
args.push('--nsInclude', `${
|
|
169
|
+
args.push('--nsInclude', `${d}.models`);
|
|
169
170
|
} else {
|
|
170
171
|
// mongorestore accepte plusieurs fois l'option --nsInclude
|
|
171
|
-
args.push('--nsInclude', `${
|
|
172
|
-
args.push('--nsInclude', `${
|
|
172
|
+
args.push('--nsInclude', `${d}.datas`);
|
|
173
|
+
args.push('--nsInclude', `${d}.models`);
|
|
173
174
|
}
|
|
174
175
|
// Le répertoire source est le dernier argument
|
|
175
176
|
args.push(restoreSourceDir);
|
|
@@ -242,6 +243,8 @@ export const dumpUserData = async (user) => {
|
|
|
242
243
|
const backupFrequency = await engine.userProvider.getBackupFrequency(user);
|
|
243
244
|
logger.info(`Fréquence de sauvegarde : ${backupFrequency}.`);
|
|
244
245
|
|
|
246
|
+
|
|
247
|
+
const d = Config.Get('dbName', dbName);
|
|
245
248
|
const collections = await MongoDatabase.listCollections().toArray();
|
|
246
249
|
for (const collection of collections) {
|
|
247
250
|
const collsToBackup = [await getUserCollectionName(user), 'models'];
|
|
@@ -249,7 +252,7 @@ export const dumpUserData = async (user) => {
|
|
|
249
252
|
const query = {_user: user.username};
|
|
250
253
|
const args = [
|
|
251
254
|
'--uri', dbUrl,
|
|
252
|
-
'--db',
|
|
255
|
+
'--db', d,
|
|
253
256
|
'--out', localTempDumpDir,
|
|
254
257
|
'--collection', collection.name,
|
|
255
258
|
'--query', JSON.stringify(query)
|
|
@@ -258,9 +261,9 @@ export const dumpUserData = async (user) => {
|
|
|
258
261
|
await execFileAsync('mongodump', args);
|
|
259
262
|
}
|
|
260
263
|
}
|
|
261
|
-
const dumpSourceDir = path.join(localTempDumpDir,
|
|
264
|
+
const dumpSourceDir = path.join(localTempDumpDir, d);
|
|
262
265
|
if (fs.existsSync(dumpSourceDir)) {
|
|
263
|
-
await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [
|
|
266
|
+
await tar.create({gzip: true, file: localArchivePath, C: localTempDumpDir}, [d]);
|
|
264
267
|
logger.info(`Archive de sauvegarde locale créée : ${localArchivePath}`);
|
|
265
268
|
} else {
|
|
266
269
|
logger.warn(`Le répertoire de dump ${dumpSourceDir} était vide. Aucune archive n'a créée.`);
|
package/src/modules/data/data.js
CHANGED
|
@@ -2,7 +2,7 @@ import {Logger} from "../../gameObject.js";
|
|
|
2
2
|
import {mkdir} from 'node:fs/promises';
|
|
3
3
|
import {onInit as historyInit} from "./data.history.js";
|
|
4
4
|
import {isDemoUser, isLocalUser} from "../../data.js";
|
|
5
|
-
import {install, maxRequestData, storageSafetyMargin} from "../../constants.js";
|
|
5
|
+
import {install, maxAlertsPerUser, maxRequestData, storageSafetyMargin} from "../../constants.js";
|
|
6
6
|
import {createCollection, getCollection} from "../mongodb.js";
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import {isGUID, sequential} from "../../core.js";
|
|
@@ -22,6 +22,7 @@ import {validateField, onInit as validationInit} from "./data.validation.js";
|
|
|
22
22
|
import {cancelAlerts, scheduleAlerts, onInit as scheduleInit} from "./data.scheduling.js";
|
|
23
23
|
import {deleteData, installPack, onInit as operationsInit} from "./data.operations.js";
|
|
24
24
|
import {jobDumpUserData, onInit as backupInit} from "./data.backup.js";
|
|
25
|
+
import {Config} from "../../config.js";
|
|
25
26
|
|
|
26
27
|
let engine;
|
|
27
28
|
let logger;
|
|
@@ -54,7 +55,8 @@ export async function onInit(defaultEngine) {
|
|
|
54
55
|
|
|
55
56
|
let modelsCollection, datasCollection, filesCollection, packsCollection, magnetsCollection, historyCollection;
|
|
56
57
|
|
|
57
|
-
|
|
58
|
+
const i = Config.Get('install', install);
|
|
59
|
+
if( i ) {
|
|
58
60
|
datasCollection = await createCollection("datas");
|
|
59
61
|
historyCollection = await createCollection("history");
|
|
60
62
|
filesCollection = await createCollection("files");
|
|
@@ -220,8 +222,9 @@ export async function checkServerCapacity(incomingDataSize = 0) {
|
|
|
220
222
|
const diskSpace = await checkDiskSpace(DATA_STORAGE_PATH);
|
|
221
223
|
const { free, size } = diskSpace;
|
|
222
224
|
|
|
225
|
+
const storageMargin = Config.Get('storageSafetyMargin', storageSafetyMargin);
|
|
223
226
|
// Limite maximale d'utilisation du disque (ex: 90% de la taille totale)
|
|
224
|
-
const maxAllowedUsage = size *
|
|
227
|
+
const maxAllowedUsage = size * storageMargin;
|
|
225
228
|
const currentUsage = size - free;
|
|
226
229
|
const projectedUsage = currentUsage + incomingDataSize;
|
|
227
230
|
|