data-primals-engine 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/client/README.md +113 -5
- package/client/src/App.jsx +17 -15
- package/client/src/constants.js +1 -1
- package/package.json +6 -3
- package/src/modules/workflow.js +72 -7
package/README.md
CHANGED
|
@@ -43,6 +43,21 @@ Possibly create a `.env` file:
|
|
|
43
43
|
```env
|
|
44
44
|
MONGO_DB_URL=mongodb://127.0.0.1:27017
|
|
45
45
|
```
|
|
46
|
+
| Variable | Description | Example |
|
|
47
|
+
|:----------------------|:------------------------------------------------------------------------|:-----------------------------------------|
|
|
48
|
+
| MONGO_DB_URL | Connection URL for your MongoDB database. | mongodb://user:pass@host:27017/db |
|
|
49
|
+
| PORT | Port on which the Express server will listen. | 7633 |
|
|
50
|
+
| JWT_SECRET | Secret key for signing JWT authentication tokens. | a_long_random_secret_string |
|
|
51
|
+
| OPENAI_API_KEY | Your optional OpenAI API key for AI features. | sk-xxxxxxxxxxxxxxxxxxxx |
|
|
52
|
+
| GOOGLE_API_KEY | Your optional Google (Gemini) API key for AI features. | AIzaSyxxxxxxxxxxxxxxxxxxxx |
|
|
53
|
+
| AWS_ACCESS_KEY_ID | AWS access key for S3 storage (files, backups). Keep empty to disable | AKIAIOSFODNN7EXAMPLE |
|
|
54
|
+
| AWS_SECRET_ACCESS_KEY | AWS secret access key. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | |
|
|
55
|
+
| AWS_REGION | Region for your S3 bucket. | eu-west-3 | |
|
|
56
|
+
| AWS_S3_BUCKET_NAME | Name of the S3 bucket to use. | my-backup-bucket | |
|
|
57
|
+
| SMTP_HOST | SMTP server host for sending emails. | smtp.example.com |
|
|
58
|
+
| SMTP_PORT | SMTP server port. | 587 |
|
|
59
|
+
| SMTP_USER | Username for SMTP authentication. | user@example.com |
|
|
60
|
+
| SMTP_PASS | Password for SMTP authentication. | password |
|
|
46
61
|
|
|
47
62
|
Start the server:
|
|
48
63
|
```bash
|
|
@@ -73,6 +88,27 @@ Define schemas using JSON:
|
|
|
73
88
|
]
|
|
74
89
|
}
|
|
75
90
|
```
|
|
91
|
+
| Type | Description | Properties/Notes |
|
|
92
|
+
|:------------|:------------------------------------------------------------------------------------|:--------------------------------------------------------------------------|
|
|
93
|
+
| string | Character string. | minLength, maxLength |
|
|
94
|
+
| string_t | International character string ID. | same as string, translated in { key, value } |
|
|
95
|
+
| number | Numeric value (integer or float). | min, max |
|
|
96
|
+
| boolean | Boolean value (true/false). | – |
|
|
97
|
+
| date | Stores a ISO date. | – |
|
|
98
|
+
| datetime | Stores an ISO date and time. | – |
|
|
99
|
+
| richtext | Rich text field (HTML) for WYSIWYG editors. | |
|
|
100
|
+
| richtext_t | International rich text field (HTML) for WYSIWYG editors. | i18n |
|
|
101
|
+
| email | String validated as an email address. | – |
|
|
102
|
+
| password | String that will be automatically hashed. | – |
|
|
103
|
+
| enum | Allows selecting a value from a predefined list. | items: ["value1", "value2"] |
|
|
104
|
+
| relation | Creates a link to a document in another model. | relation: "target_model_name", multiple: true/false |
|
|
105
|
+
| file | For uploading a file (stored on S3 if configured). | allowedTypes:['image/jpeg', 'image/png', 'image/bmp'], maxSize: 1024*1000 |
|
|
106
|
+
| image | Specialized file type for images, with preview. | – |
|
|
107
|
+
| array | Stores a list of values. | itemsType: 'enum' // any type except relations |
|
|
108
|
+
| object | Stores a nested JSON object. – | |
|
|
109
|
+
| code | Stores language="*" as string, stores language="json" as arbitrary JSON structure. | language="json" conditionBuilder=true |
|
|
110
|
+
| model | Stores a model by name | – |
|
|
111
|
+
| modelField | Stores a model field path | – |
|
|
76
112
|
|
|
77
113
|
### 2. Modules
|
|
78
114
|
Activatable features:
|
package/client/README.md
CHANGED
|
@@ -1,8 +1,116 @@
|
|
|
1
|
-
|
|
1
|
+
This guide explains how to integrate and use the client-side part of the `data-primals-engine` library in an external React application (such as hackersonline).
|
|
2
|
+
The library provides a set of context Providers, Hooks, and UI components to interact with the `data-primals-engine` backend.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
+
# Table of Contents
|
|
5
|
+
## Prerequisites
|
|
6
|
+
Your React application must have the following dependencies installed:
|
|
7
|
+
- react & react-dom (version >= 18.0.0)
|
|
8
|
+
- react-query (version >= 3.0.0)
|
|
9
|
+
- react-router-dom
|
|
10
|
+
- react-cookie
|
|
4
11
|
|
|
5
|
-
|
|
12
|
+
## Installation
|
|
13
|
+
Install the library:
|
|
14
|
+
```shell
|
|
15
|
+
npm install ../../data-primals-engine
|
|
16
|
+
```
|
|
17
|
+
Install peer dependencies:
|
|
18
|
+
```shell
|
|
19
|
+
npm install react-query react-router-dom react-cookie
|
|
20
|
+
```
|
|
6
21
|
|
|
7
|
-
|
|
8
|
-
|
|
22
|
+
## Basic Setup
|
|
23
|
+
|
|
24
|
+
The setup mainly occurs in your application's root file (typically App.jsx or main.jsx).
|
|
25
|
+
|
|
26
|
+
It consists of initializing a single QueryClient instance and wrapping your app with the required Providers.
|
|
27
|
+
|
|
28
|
+
### Set the QueryClient (Singleton)
|
|
29
|
+
For react-query caching to work correctly across your app (including inside the library), it is crucial to instantiate QueryClient only once.
|
|
30
|
+
|
|
31
|
+
The data-primals-engine library needs to know which QueryClient instance you're using. It exports a setQueryClient function for that purpose.
|
|
32
|
+
|
|
33
|
+
### Set up the Providers
|
|
34
|
+
Your app must be wrapped by several Providers in the correct order so the contexts are available to all child components.
|
|
35
|
+
|
|
36
|
+
## Full example (App.jsx)
|
|
37
|
+
Here's what your main application file should look like:
|
|
38
|
+
```jsx
|
|
39
|
+
// In your main App.jsx file (e.g., hackersonline/client/src/App.jsx)
|
|
40
|
+
|
|
41
|
+
import React from 'react';
|
|
42
|
+
import { BrowserRouter, Routes, Route } from 'react-router-dom';
|
|
43
|
+
import { CookiesProvider } from 'react-cookie';
|
|
44
|
+
|
|
45
|
+
// 1. Import QueryClient and its Provider from react-query
|
|
46
|
+
import { QueryClient, QueryClientProvider } from 'react-query';
|
|
47
|
+
|
|
48
|
+
// 2. Import Providers and the config function from data-primals-engine
|
|
49
|
+
import {
|
|
50
|
+
ModelProvider,
|
|
51
|
+
AuthProvider,
|
|
52
|
+
UIProvider,
|
|
53
|
+
NotificationProvider,
|
|
54
|
+
setQueryClient // <-- Important config function
|
|
55
|
+
} from 'data-primals-engine/client';
|
|
56
|
+
|
|
57
|
+
// 3. Import the i18n config (see dedicated section)
|
|
58
|
+
import 'data-primals-engine/i18n';
|
|
59
|
+
|
|
60
|
+
// 4. (CRUCIAL) Create a unique instance of QueryClient
|
|
61
|
+
const queryClient = new QueryClient();
|
|
62
|
+
|
|
63
|
+
// 5. (CRUCIAL) Pass the instance to the shared library
|
|
64
|
+
setQueryClient(queryClient);
|
|
65
|
+
|
|
66
|
+
// Your page components
|
|
67
|
+
import HomePage from './pages/HomePage';
|
|
68
|
+
import UserDashboard from './pages/UserDashboard';
|
|
69
|
+
|
|
70
|
+
function App() {
|
|
71
|
+
return (
|
|
72
|
+
// 6. Wrap the app with all required providers
|
|
73
|
+
<QueryClientProvider client={queryClient}>
|
|
74
|
+
<AuthProvider>
|
|
75
|
+
<CookiesProvider>
|
|
76
|
+
<ModelProvider>
|
|
77
|
+
<BrowserRouter>
|
|
78
|
+
<UIProvider>
|
|
79
|
+
<NotificationProvider>
|
|
80
|
+
|
|
81
|
+
{/* Your app content */}
|
|
82
|
+
<Routes>
|
|
83
|
+
<Route path="/" element={<HomePage />} />
|
|
84
|
+
<Route path="/dashboard" element={<UserDashboard />} />
|
|
85
|
+
{/* ... other routes */}
|
|
86
|
+
</Routes>
|
|
87
|
+
|
|
88
|
+
</NotificationProvider>
|
|
89
|
+
</UIProvider>
|
|
90
|
+
</BrowserRouter>
|
|
91
|
+
</ModelProvider>
|
|
92
|
+
</CookiesProvider>
|
|
93
|
+
</AuthProvider>
|
|
94
|
+
</QueryClientProvider>
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export default App;
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Internationalization (i18n)
|
|
102
|
+
The library comes with its own i18next configuration and translation files.
|
|
103
|
+
To enable it in your app, you just need to import the config module once in your main file (App.jsx or main.jsx):
|
|
104
|
+
|
|
105
|
+
```jsx
|
|
106
|
+
// In App.jsx or main.jsx
|
|
107
|
+
import 'data-primals-engine/i18n';
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
This simple import will:
|
|
111
|
+
|
|
112
|
+
- Initialize i18next.
|
|
113
|
+
- Load default translations (French, English, etc.).
|
|
114
|
+
- Enable automatic browser language detection.
|
|
115
|
+
|
|
116
|
+
You can then use react-i18next components and hooks (useTranslation, Trans) as usual in your application.
|
package/client/src/App.jsx
CHANGED
|
@@ -55,7 +55,9 @@ import {useCookies, CookiesProvider} from "react-cookie";
|
|
|
55
55
|
import { translations as allTranslations} from "data-primals-engine/i18n";
|
|
56
56
|
import {getRandom} from "data-primals-engine/core";
|
|
57
57
|
import {getUserHash} from "data-primals-engine/data";
|
|
58
|
-
import {seoTitle} from "./constants.js";
|
|
58
|
+
import {promotionalMessages, seoTitle} from "./constants.js";
|
|
59
|
+
import i18next from "i18next";
|
|
60
|
+
import {websiteTranslations} from "./translations.js";
|
|
59
61
|
|
|
60
62
|
let queryClient = new QueryClient();
|
|
61
63
|
|
|
@@ -69,6 +71,7 @@ function TopBar({header}) {
|
|
|
69
71
|
const { i18n, t } = useTranslation();
|
|
70
72
|
const lang = (i18n.resolvedLanguage || i18n.language).split(/[-_]/)?.[0];
|
|
71
73
|
|
|
74
|
+
|
|
72
75
|
useEffect(() => {
|
|
73
76
|
if( location.pathname ){
|
|
74
77
|
gtag('config', 'G-NDHNSVB4YB', { page_path: location.pathname });
|
|
@@ -165,6 +168,14 @@ function Layout ({header, translationMutation, routes, body, footer}) {
|
|
|
165
168
|
}
|
|
166
169
|
];
|
|
167
170
|
|
|
171
|
+
const promotionalMessages = [
|
|
172
|
+
{ text: t('promo.rotation.6') },
|
|
173
|
+
{ text: t('promo.rotation.1') },
|
|
174
|
+
{ text: t('promo.rotation.2') },
|
|
175
|
+
{ text: t('promo.rotation.1') },
|
|
176
|
+
{ text: t('promo.rotation.4') },
|
|
177
|
+
{ text: t('promo.rotation.5') }
|
|
178
|
+
];
|
|
168
179
|
const { setCurrentTourSteps, setAllTourSteps, currentTour, setCurrentTour } = useUI();
|
|
169
180
|
const [translations, setTranslations] = useState([]);
|
|
170
181
|
|
|
@@ -175,13 +186,13 @@ function Layout ({header, translationMutation, routes, body, footer}) {
|
|
|
175
186
|
const changeLanguage = (newLang) => {
|
|
176
187
|
if (typeof(newLang) === 'string' && newLang) {
|
|
177
188
|
|
|
178
|
-
i18n.removeResourceBundle(lang, "translations");
|
|
179
|
-
i18n.addResourceBundle(newLang, "translations", translations);
|
|
180
189
|
i18n.changeLanguage(newLang, (err)=>{
|
|
181
190
|
|
|
191
|
+
i18next.removeResourceBundle(lang, "translation");
|
|
192
|
+
i18next.addResourceBundle(newLang, 'translation', {...websiteTranslations[newLang]['translation'], ...translations[newLang]?.['translation']});
|
|
193
|
+
|
|
182
194
|
gtag("event", "change_language "+newLang);
|
|
183
195
|
});
|
|
184
|
-
|
|
185
196
|
}
|
|
186
197
|
};
|
|
187
198
|
|
|
@@ -277,15 +288,6 @@ function Layout ({header, translationMutation, routes, body, footer}) {
|
|
|
277
288
|
const [lightboxOpened, setLightboxOpened] = React.useState(false);
|
|
278
289
|
const [lightboxIndex, setLightboxIndex] = React.useState(0);
|
|
279
290
|
|
|
280
|
-
const myMessages = [
|
|
281
|
-
{ text: t('promo.rotation.6') },
|
|
282
|
-
{ text: t('promo.rotation.1') },
|
|
283
|
-
{ text: t('promo.rotation.2') },
|
|
284
|
-
{ text: t('promo.rotation.1') },
|
|
285
|
-
{ text: t('promo.rotation.4') },
|
|
286
|
-
{ text: t('promo.rotation.5') },
|
|
287
|
-
];
|
|
288
|
-
|
|
289
291
|
const slides = [{src: "/prez1.jpg", title: t('prez1')},{src: "/prez6.jpg", title: i18n.t('prez3')},{src: "/prez2.jpg", title: t('prez2')},{src: "/prez4.jpg", title: i18n.t('prez4')},{src: "/prez5.jpg", title: i18n.t('prez5')}];
|
|
290
292
|
|
|
291
293
|
const { addNotification } = useNotificationContext();
|
|
@@ -389,8 +391,8 @@ function Layout ({header, translationMutation, routes, body, footer}) {
|
|
|
389
391
|
width={250}
|
|
390
392
|
/>
|
|
391
393
|
<div className="bubble flex-1">
|
|
392
|
-
<h1><Trans i18nKey={"prez.title"}>
|
|
393
|
-
<div className="inner"><MessageRotator messages={
|
|
394
|
+
<h1><Trans i18nKey={"prez.title"}>{seoTitle}</Trans></h1>
|
|
395
|
+
<div className="inner"><MessageRotator messages={promotionalMessages} fadeDuration={200} defaultDelay={7500}/></div>
|
|
394
396
|
</div>
|
|
395
397
|
</div>
|
|
396
398
|
<div className="flex flex-row">
|
package/client/src/constants.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "data-primals-engine",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "data-primals-engine is the package responsible from handling large amount of data in a practical and performant way. It can handle large amount of data in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation.",
|
|
5
5
|
"main": "src/engine.js",
|
|
6
6
|
"type": "module",
|
|
@@ -76,9 +76,9 @@
|
|
|
76
76
|
"node-cache": "^5.1.2",
|
|
77
77
|
"node-schedule": "^2.1.1",
|
|
78
78
|
"nodemailer": "^7.0.5",
|
|
79
|
-
"prop-types": "^15.8.1",
|
|
80
79
|
"openai": "^5.10.2",
|
|
81
80
|
"process": "^0.11.10",
|
|
81
|
+
"prop-types": "^15.8.1",
|
|
82
82
|
"randomcolor": "^0.6.2",
|
|
83
83
|
"react-helmet": "^6.1.0",
|
|
84
84
|
"react-i18next": "^15.6.1",
|
|
@@ -90,6 +90,7 @@
|
|
|
90
90
|
"tar": "^7.4.3",
|
|
91
91
|
"uniqid": "^5.4.0",
|
|
92
92
|
"vitest": "^3.2.4",
|
|
93
|
+
"vm2": "^3.9.19",
|
|
93
94
|
"yaml": "^2.8.0"
|
|
94
95
|
},
|
|
95
96
|
"devDependencies": {
|
|
@@ -99,7 +100,9 @@
|
|
|
99
100
|
"eslint": "^9.32.0",
|
|
100
101
|
"globals": "^16.3.0"
|
|
101
102
|
},
|
|
102
|
-
"subPackages": [
|
|
103
|
+
"subPackages": [
|
|
104
|
+
"client"
|
|
105
|
+
],
|
|
103
106
|
"keywords": [
|
|
104
107
|
"mongodb",
|
|
105
108
|
"data",
|
package/src/modules/workflow.js
CHANGED
|
@@ -4,17 +4,16 @@ import schedule from "node-schedule";
|
|
|
4
4
|
import {ObjectId} from "mongodb";
|
|
5
5
|
import crypto from "node:crypto";
|
|
6
6
|
|
|
7
|
+
import {VM, VMScript} from 'vm2';
|
|
7
8
|
import {Logger} from "../gameObject.js";
|
|
8
|
-
import {deleteData,
|
|
9
|
+
import {deleteData, insertData, patchData, searchData} from "./data.js";
|
|
9
10
|
import {maxExecutionsByStep, maxWorkflowSteps} from "../constants.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import {ChatOpenAI} from "@langchain/openai";
|
|
12
|
+
import {ChatGoogleGenerativeAI} from "@langchain/google-genai";
|
|
13
|
+
import {ChatPromptTemplate} from "@langchain/core/prompts";
|
|
13
14
|
import i18n from "data-primals-engine/i18n";
|
|
14
15
|
import {sendEmail} from "../email.js";
|
|
15
16
|
|
|
16
|
-
// 1. ADD THIS IMPORT AT THE TOP OF THE FILE
|
|
17
|
-
// This allows the module to call its own exported functions.
|
|
18
17
|
import * as workflowModule from './workflow.js';
|
|
19
18
|
|
|
20
19
|
let logger = null;
|
|
@@ -181,6 +180,69 @@ export async function scheduleWorkflowTriggers() {
|
|
|
181
180
|
}
|
|
182
181
|
|
|
183
182
|
|
|
183
|
+
/* Remplacement de la fonction executeSafeJavascript
|
|
184
|
+
async function executeWithIsolatedVm(actionDef, context) {
|
|
185
|
+
const code = actionDef.script;
|
|
186
|
+
const isolate = new ivm.Isolate({ memoryLimit: 128 }); // Limite de 128MB de mémoire
|
|
187
|
+
|
|
188
|
+
try {
|
|
189
|
+
const vmContext = await isolate.createContext();
|
|
190
|
+
const jail = vmContext.global;
|
|
191
|
+
|
|
192
|
+
// On expose une fonction 'log' sécurisée à l'intérieur de la VM.
|
|
193
|
+
// C'est une référence, pas la fonction elle-même.
|
|
194
|
+
await jail.set('log', new ivm.Reference(function(...args) {
|
|
195
|
+
// On peut préfixer les logs pour savoir qu'ils viennent de la VM
|
|
196
|
+
logger.info('[VM Script Log]', ...args);
|
|
197
|
+
}));
|
|
198
|
+
|
|
199
|
+
// On injecte les données du contexte. Elles sont COPIÉES, pas référencées.
|
|
200
|
+
// C'est une caractéristique de sécurité clé.
|
|
201
|
+
await jail.set('contextData', new ivm.ExternalCopy(context).copyInto());
|
|
202
|
+
// On peut aussi exposer des fonctions spécifiques de votre application
|
|
203
|
+
// await jail.set('myApiFunction', new ivm.Reference(async (params) => { ... }));
|
|
204
|
+
|
|
205
|
+
// On compile le script
|
|
206
|
+
const script = await isolate.compileScript(code);
|
|
207
|
+
|
|
208
|
+
// On exécute le script avec un timeout
|
|
209
|
+
const result = await script.run(vmContext, { timeout: 1000 });
|
|
210
|
+
|
|
211
|
+
// Le résultat est une référence, on le copie pour l'utiliser dans notre contexte principal.
|
|
212
|
+
return result;
|
|
213
|
+
|
|
214
|
+
} catch (error) {
|
|
215
|
+
logger.error("Error executing script with isolated-vm:", error.stack);
|
|
216
|
+
// On propage une erreur propre
|
|
217
|
+
throw new Error(`Script execution failed: ${error.message}`);
|
|
218
|
+
} finally {
|
|
219
|
+
// TRÈS IMPORTANT : Toujours libérer l'isolate pour éviter les fuites de mémoire.
|
|
220
|
+
if (isolate && !isolate.isDisposed) {
|
|
221
|
+
isolate.dispose();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
REQUIRES NODE >=20
|
|
226
|
+
*/
|
|
227
|
+
async function executeSafeJavascript(actionDef, context) {
|
|
228
|
+
const code = actionDef.script;
|
|
229
|
+
const vm = new VM({
|
|
230
|
+
timeout: 1000, // Time out after 1 second
|
|
231
|
+
sandbox: context, // Pass the context object
|
|
232
|
+
console: 'redirect', // Redirect console output
|
|
233
|
+
require: false, // Disable require
|
|
234
|
+
wasm: false // disable WebAssembly
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
try {
|
|
238
|
+
const script = new VMScript(code);
|
|
239
|
+
return vm.run(script);
|
|
240
|
+
} catch (error) {
|
|
241
|
+
console.error("Error executing script:", error);
|
|
242
|
+
throw error; // or return an error object
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
184
246
|
/**
|
|
185
247
|
* Handles the 'Webhook' workflow action.
|
|
186
248
|
* Sends an HTTP request to a specified URL with substituted data using native fetch.
|
|
@@ -671,7 +733,10 @@ export async function executeStepAction(actionDef, contextData, user, dbCollecti
|
|
|
671
733
|
case 'SendEmail':
|
|
672
734
|
result = await handleSendEmailAction(actionDef, contextData, user);
|
|
673
735
|
break;
|
|
674
|
-
|
|
736
|
+
case 'ExecuteScript':
|
|
737
|
+
result = await executeSafeJavascript(actionDef, contextData);
|
|
738
|
+
break;
|
|
739
|
+
|
|
675
740
|
// ... autres cases à venir ...
|
|
676
741
|
default:
|
|
677
742
|
logger.error(`[executeStepAction] Unknown action type: ${actionDef.type}`);
|