data-primals-engine 1.5.1 → 1.5.2

package/src/modules/data/data.relations.js

@@ -2,7 +2,7 @@ import {getDefaultForType, getFieldValueHash} from "../../data.js";
 import {Event} from "../../events.js";
 import {getCollectionForUser, isObjectId} from "../mongodb.js";
 import {ObjectId} from "mongodb";
-import {isPlainObject} from "../../core.js";
+import {isPlainObject, parseSafeJSON} from "../../core.js";
 import {dataTypes, getModel, searchData} from "./data.operations.js";
 import {validateModelData} from "./data.validation.js";
 import i18n from "../../i18n.js";

package/src/modules/data/data.routes.js

@@ -12,7 +12,7 @@ import {
     maxModelsPerUser, maxPackData, maxPackPreviewData
 } from "../../constants.js";
 import {datasCollection, getCollection, getCollectionForUser, isObjectId, modelsCollection} from "../mongodb.js";
-import {countKeys, safeAssignObject, uuidv4} from "../../core.js";
+import {countKeys, parseSafeJSON, safeAssignObject, uuidv4} from "../../core.js";
 import {Event} from "../../events.js";
 import fs from "node:fs";
 import i18n from "../../i18n.js";
@@ -1192,7 +1192,7 @@ export async function registerRoutes(defaultEngine){
                     totalCount = result.length > 0 ? result[0]['count'] : 0;
                 } catch (totalError) {
                     if (totalError instanceof SyntaxError) {
-                        console.error(`>>> ERREUR JSON.parse (totalMatchFormula) pour KPI ${id}:`, kpiDef.totalMatchFormula, totalError);
+                        console.error(`>>> ERREUR parseSafeJSON (totalMatchFormula) pour KPI ${id}:`, kpiDef.totalMatchFormula, totalError);
                     } else {
                         console.error(`>>> ERREUR Aggregate (totalCount) pour KPI ${id}:`, totalError);
                     }
@@ -1206,7 +1206,7 @@ export async function registerRoutes(defaultEngine){
                     const parsedMatch = kpiDef.matchFormula || {}; // Assurer un objet vide
                     matchFilter = { ...matchFilter, ...parsedMatch };
                 } catch (matchError) {
-                    console.error(`>>> ERREUR JSON.parse (matchFormula) pour KPI ${id}:`, kpiDef.matchFormula, matchError);
+                    console.error(`>>> ERREUR parseSafeJSON (matchFormula) pour KPI ${id}:`, kpiDef.matchFormula, matchError);
                     throw matchError; // Relancer
                 }
             }

package/src/modules/user.js

@@ -19,6 +19,7 @@ import {Config} from "../config.js";
 
 export const userInitiator = async (req, res, next) => {
 
+    const engine = req.app.get('engine');
     const lang = getAPILang(req.query.lang || req.headers['Accept-Language']);
 
     req.lang = lang;

package/src/modules/workflow.js

@@ -19,7 +19,7 @@ import {getEnv, getSmtpConfig} from "./user.js";
 import {getHost} from "../constants.js";
 import {providers} from "./assistant/constants.js";
 import {getAIProvider} from "./assistant/assistant.js";
-import { safeAssignObject} from "../core.js";
+import {parseSafeJSON, safeAssignObject} from "../core.js";
 import {Config} from "../config.js";
 
 let logger = null;
@@ -1655,7 +1655,7 @@ async function executeGenerateAIContentAction(action, context, user) {
     }
 
     // 2. Initialize the LLM client with LangChain
-    let llm = getAIProvider(aiProvider, aiModel, apiKey);
+    let llm = await getAIProvider(aiProvider, aiModel, apiKey);
     if( !llm ) {
         const message = `Failed to initialize AI client for ${aiProvider}: ${initError.message}`;
         logger.error(`[AI Action] ${message}`);

package/src/openai.jobs.js

@@ -1,4 +1,5 @@
 import OpenAI from "openai";
+import {parseSafeJSON} from "./core.js";
 
 
 let engine = null;
@@ -132,8 +133,8 @@ export const openaiJobModel = async (lang, txt, history, existingModels = []) =>
         const aiResponse = completion.choices[0].message.content;
 
         try {
-            // JSON.parse fonctionnera que le contenu soit un objet ou un tableau
-            return JSON.parse(aiResponse); // On retourne directement le résultat parsé
+            // parseSafeJSON fonctionnera que le contenu soit un objet ou un tableau
+            return parseSafeJSON(aiResponse); // On retourne directement le résultat parsé
         } catch (e) {
             console.error("Erreur de parsing du JSON de l'IA:", e);
             throw new Error("Réponse invalide de l'IA.");

package/src/sso.js

@@ -1,6 +1,6 @@
 import passport from 'passport';
 import session from 'express-session';
-import { Behaviour } from './gameObject.js';
+import {Behaviour, Logger} from './gameObject.js';
 import { cookiesSecret } from "./constants.js";
 import {UserProvider} from "./providers.js";
 import {getCollection} from "./modules/mongodb.js";
@@ -110,7 +110,7 @@ export class Sso extends Behaviour {
             throw new Error("PassportAuth component requires an ssoUserProvider to be initialized.");
         }
         this.#ssoUserProvider = ssoUserProvider;
-        this.#logger = this.gameObject.getComponent('Logger'); // Assumant que Logger est un composant
+        this.#logger = this.gameObject.getComponent(Logger); // Assumant que Logger est un composant
 
         const app = this.gameObject; // this.gameObject est l'instance de l'engine (Express app)
 

package/src/workers/import-export-worker.js

@@ -1,5 +1,5 @@
 import { parentPort } from 'worker_threads';
-import { parse } from 'csv-parse/sync'; // On utilise la version synchrone, c'est ok dans un worker
+import { parse } from 'csv-parse/sync';
 
 parentPort.on('message', ({ action, payload }) => {
     try {

package/test/data.history.integration.test.js

@@ -1,193 +1,265 @@
-// __tests__/data.history.integration.test.js
-import { ObjectId } from 'mongodb';
-import { expect, describe, it, beforeAll, afterAll, beforeEach } from 'vitest';
-import { Config } from '../src/config.js';
-import { sleep } from '../src/core.js';
-import { insertData, editData, deleteModels } from '../src/index.js';
-import { getCollection, getCollectionForUser } from '../src/modules/mongodb.js';
-import { generateUniqueName, initEngine } from "../src/setenv.js";
-import {purgeData} from "../src/modules/data/data.history.js";
-import {MongoDatabase} from "../src/engine.js";
-
-let engine;
-let testUser;
-let historyCollection;
-let datasCollection;
-let modelsCollection;
-
-describe('Data History Module Integration Tests', () => {
-
-    beforeAll(async () => {
-        // IMPORTANT: Add the history module to the engine configuration for tests
-        Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow", "user", "assistant"]);
-        engine = await initEngine();
-
-        // Setup a single user for all tests in this suite
-        testUser = {
-            username: generateUniqueName('testUserHistory'),
-            userPlan: 'free',
-            email: generateUniqueName('test') + '@example.com'
-        };
-
-        // Initialize collection instances
-        historyCollection = getCollection('history');
-        datasCollection = await getCollectionForUser(testUser);
-        modelsCollection = getCollection('models');
-    });
-
-    afterAll(async () =>{
-        await purgeData(testUser);
-        await datasCollection.drop();
-    })
-
-    // Clean up collections before each test to ensure isolation
-    beforeEach(async () => {
-        await historyCollection.deleteMany({ 'user.username': testUser.username });
-        await datasCollection.deleteMany({ _user: testUser.username });
-        await modelsCollection.deleteMany({ _user: testUser.username });
-    });
-
-    afterAll(async () => {
-        // Final cleanup
-        await historyCollection.deleteMany({ 'user.username': testUser.username });
-        await datasCollection.deleteMany({ _user: testUser.username });
-        await modelsCollection.deleteMany({ _user: testUser.username });
-    });
-
-    it('should create a full snapshot history record on document creation', async () => {
-        // 1. Define and create a model with history enabled
-        const modelName = generateUniqueName('productHistory');
-        const productModelDef = {
-            name: modelName,
-            description:"",
-            _user: testUser.username,
-            history: {
-                enabled: true,
-                // For creation, all fields are snapshotted regardless of this config
-                fields: {
-                    price: true,
-                    stock: true
-                }
-            },
-            fields: [
-                { name: 'name', type: 'string', required: true },
-                { name: 'price', type: 'number' },
-                { name: 'stock', type: 'number' },
-                { name: 'description', type: 'string' }
-            ]
-        };
-        await modelsCollection.insertOne(productModelDef);
-
-        // 2. Insert a new document
-        const initialData = {
-            name: 'Super Widget',
-            price: 99.99,
-            stock: 100,
-            description: 'A very super widget.'
-        };
-        const insertResult = await insertData(modelName, initialData, {}, testUser);
-        expect(insertResult.success).toBe(true);
-        const docId = new ObjectId(insertResult.insertedIds[0]);
-
-        // 3. Verify the history record
-        const historyRecord = await historyCollection.findOne({ documentId: new ObjectId(docId) });
-
-        expect(historyRecord).not.toBeNull();
-        expect(historyRecord.documentId.toString()).toBe(docId.toString());
-        expect(historyRecord.model).toBe(modelName);
-        expect(historyRecord.version).toBe(1);
-        expect(historyRecord.operation).toBe('create');
-        expect(historyRecord.user.username).toBe(testUser.username);
-
-        // 4. Verify the snapshot
-        expect(historyRecord.snapshot).not.toBeNull();
-        expect(historyRecord.snapshot.name).toBe('Super Widget');
-        expect(historyRecord.snapshot.price).toBe(99.99);
-        expect(historyRecord.snapshot.stock).toBe(100);
-        expect(historyRecord.snapshot.description).toBe('A very super widget.');
-        expect(historyRecord.changes).toBeUndefined(); // No 'changes' field on creation
-    });
-
-    it('should create a diff history record on document update for historized fields only', async () => {
-        // 1. Define and create a model with specific history fields
-        const modelName = generateUniqueName('productHistorySelective');
-        const productModelDef = {
-            name: modelName,
-            description:"",
-            _user: testUser.username,
-            history: {
-                enabled: true,
-                fields: {
-                    price: true, // Track this
-                    stock: true  // Track this
-                    // 'description' is NOT tracked
-                }
-            },
-            fields: [
-                { name: 'name', type: 'string', required: true },
-                { name: 'price', type: 'number' },
-                { name: 'stock', type: 'number' },
-                { name: 'description', type: 'string' }
-            ]
-        };
-        await modelsCollection.insertOne(productModelDef);
-
-        // 2. Insert the initial document
-        const initialData = { name: 'Selective Widget', price: 50, stock: 200, description: 'Initial description.' };
-        const insertResult = await insertData(modelName, initialData, {}, testUser);
-        const docId = new ObjectId(insertResult.insertedIds[0]);
-
-        // 3. Edit the document: change one historized field and one non-historized field
-        const updateData = { price: 55.5, description: 'Updated description.' };
-        const editResult = await editData(modelName, { _id: docId }, updateData, {}, testUser);
-        expect(editResult.success).toBe(true);
-
-        // 4. Verify the new history record (v2)
-        const historyRecord = await historyCollection.findOne({ documentId: docId, version: 2 });
-
-        expect(historyRecord).not.toBeNull();
-        expect(historyRecord.operation).toBe('update');
-        expect(historyRecord.snapshot).toBeUndefined(); // No 'snapshot' on update
-
-        // 5. Verify the 'changes' object
-        const changes = historyRecord.changes;
-        expect(changes).not.toBeNull();
-        expect(changes.price).toBeDefined();
-        expect(changes.price.from).toBe(50);
-        expect(changes.price.to).toBe(55.5);
-        expect(changes.description).toBeUndefined();
-        expect(changes.stock).toBeUndefined();
-    });
-
-    it('should NOT create a history record if only non-historized fields are updated', async () => {
-        // 1. Setup model
-        const modelName = generateUniqueName('productHistoryNoOp');
-        const productModelDef = {
-            name: modelName,
-            description:"",
-            _user: testUser.username,
-            history: { enabled: true, fields: { price: true } },
-            fields: [
-                { name: 'name', type: 'string', required: true },
-                { name: 'price', type: 'number' },
-                { name: 'description', type: 'string' }
-            ]
-        };
-        await modelsCollection.insertOne(productModelDef);
-
-        // 2. Insert initial document
-        const initialData = { name: 'No-Op Widget', price: 10, description: 'Initial.' };
-        const insertResult = await insertData(modelName, initialData, {}, testUser);
-        const docId = new ObjectId(insertResult.insertedIds[0]);
-
-        // 3. Edit ONLY a non-historized field
-        const updateData = { description: 'This change should not be recorded.' };
-        await editData(modelName, { _id: docId }, updateData, {}, testUser);
-
-        await sleep(2000);
-
-        // 4. Verify that NO new history record was created (only the 'create' record exists)
-        const historyCount = await historyCollection.countDocuments({ documentId: docId });
-        expect(historyCount).toBe(1);
-    });
+// __tests__/data.history.integration.test.js
+import { ObjectId } from 'mongodb';
+import { expect, describe, it, beforeAll, afterAll, beforeEach } from 'vitest';
+import { handleGetHistoryRequest } from '../src/modules/data/data.history.js';
+import { Config } from '../src/config.js';
+import { sleep } from '../src/core.js';
+import { insertData, editData, deleteModels } from '../src/index.js';
+import { getCollection, getCollectionForUser } from '../src/modules/mongodb.js';
+import { generateUniqueName, initEngine } from "../src/setenv.js";
+import {purgeData} from "../src/modules/data/data.history.js";
+import {MongoDatabase} from "../src/engine.js";
+
+let engine;
+let testUser;
+let historyCollection;
+let datasCollection;
+let modelsCollection;
+
+describe('Data History Module Integration Tests', () => {
+
+    beforeAll(async () => {
+        // IMPORTANT: Add the history module to the engine configuration for tests
+        Config.Set("modules", ["mongodb", "data", "file", "bucket", "workflow", "user", "assistant"]);
+        engine = await initEngine();
+
+        // Setup a single user for all tests in this suite
+        testUser = {
+            username: generateUniqueName('testUserHistory'),
+            userPlan: 'free',
+            email: generateUniqueName('test') + '@example.com'
+        };
+
+        // Initialize collection instances
+        historyCollection = getCollection('history');
+        datasCollection = await getCollectionForUser(testUser);
+        modelsCollection = getCollection('models');
+    });
+
+    afterAll(async () =>{
+        await purgeData(testUser);
+        await datasCollection.drop();
+    })
+
+    // Clean up collections before each test to ensure isolation
+    beforeEach(async () => {
+        await historyCollection.deleteMany({ 'user.username': testUser.username });
+        await datasCollection.deleteMany({ _user: testUser.username });
+        await modelsCollection.deleteMany({ _user: testUser.username });
+    });
+
+    afterAll(async () => {
+        // Final cleanup
+        await historyCollection.deleteMany({ 'user.username': testUser.username });
+        await datasCollection.deleteMany({ _user: testUser.username });
+        await modelsCollection.deleteMany({ _user: testUser.username });
+    });
+
+    it('should create a full snapshot history record on document creation', async () => {
+        // 1. Define and create a model with history enabled
+        const modelName = generateUniqueName('productHistory');
+        const productModelDef = {
+            name: modelName,
+            description:"",
+            _user: testUser.username,
+            history: {
+                enabled: true,
+                // For creation, all fields are snapshotted regardless of this config
+                fields: {
+                    price: true,
+                    stock: true
+                }
+            },
+            fields: [
+                { name: 'name', type: 'string', required: true },
+                { name: 'price', type: 'number' },
+                { name: 'stock', type: 'number' },
+                { name: 'description', type: 'string' }
+            ]
+        };
+        await modelsCollection.insertOne(productModelDef);
+
+        // 2. Insert a new document
+        const initialData = {
+            name: 'Super Widget',
+            price: 99.99,
+            stock: 100,
+            description: 'A very super widget.'
+        };
+        const insertResult = await insertData(modelName, initialData, {}, testUser);
+        expect(insertResult.success).toBe(true);
+        const docId = new ObjectId(insertResult.insertedIds[0]);
+
+        // 3. Verify the history record
+        const historyRecord = await historyCollection.findOne({ documentId: new ObjectId(docId) });
+
+        expect(historyRecord).not.toBeNull();
+        expect(historyRecord.documentId.toString()).toBe(docId.toString());
+        expect(historyRecord.model).toBe(modelName);
+        expect(historyRecord.version).toBe(1);
+        expect(historyRecord.operation).toBe('create');
+        expect(historyRecord.user.username).toBe(testUser.username);
+
+        // 4. Verify the snapshot
+        expect(historyRecord.snapshot).not.toBeNull();
+        expect(historyRecord.snapshot.name).toBe('Super Widget');
+        expect(historyRecord.snapshot.price).toBe(99.99);
+        expect(historyRecord.snapshot.stock).toBe(100);
+        expect(historyRecord.snapshot.description).toBe('A very super widget.');
+        expect(historyRecord.changes).toBeUndefined(); // No 'changes' field on creation
+    });
+
+    it('should create a diff history record on document update for historized fields only', async () => {
+        // 1. Define and create a model with specific history fields
+        const modelName = generateUniqueName('productHistorySelective');
+        const productModelDef = {
+            name: modelName,
+            description:"",
+            _user: testUser.username,
+            history: {
+                enabled: true,
+                fields: {
+                    price: true, // Track this
+                    stock: true  // Track this
+                    // 'description' is NOT tracked
+                }
+            },
+            fields: [
+                { name: 'name', type: 'string', required: true },
+                { name: 'price', type: 'number' },
+                { name: 'stock', type: 'number' },
+                { name: 'description', type: 'string' }
+            ]
+        };
+        await modelsCollection.insertOne(productModelDef);
+
+        // 2. Insert the initial document
+        const initialData = { name: 'Selective Widget', price: 50, stock: 200, description: 'Initial description.' };
+        const insertResult = await insertData(modelName, initialData, {}, testUser);
+        const docId = new ObjectId(insertResult.insertedIds[0]);
+
+        // 3. Edit the document: change one historized field and one non-historized field
+        const updateData = { price: 55.5, description: 'Updated description.' };
+        const editResult = await editData(modelName, { _id: docId }, updateData, {}, testUser);
+        expect(editResult.success).toBe(true);
+
+        // 4. Verify the new history record (v2)
+        const historyRecord = await historyCollection.findOne({ documentId: docId, version: 2 });
+
+        expect(historyRecord).not.toBeNull();
+        expect(historyRecord.operation).toBe('update');
+        expect(historyRecord.snapshot).toBeUndefined(); // No 'snapshot' on update
+
+        // 5. Verify the 'changes' object
+        const changes = historyRecord.changes;
+        expect(changes).not.toBeNull();
+        expect(changes.price).toBeDefined();
+        expect(changes.price.from).toBe(50);
+        expect(changes.price.to).toBe(55.5);
+        expect(changes.description).toBeUndefined();
+        expect(changes.stock).toBeUndefined();
+    });
+
+    it('should NOT create a history record if only non-historized fields are updated', async () => {
+        // 1. Setup model
+        const modelName = generateUniqueName('productHistoryNoOp');
+        const productModelDef = {
+            name: modelName,
+            description:"",
+            _user: testUser.username,
+            history: { enabled: true, fields: { price: true } },
+            fields: [
+                { name: 'name', type: 'string', required: true },
+                { name: 'price', type: 'number' },
+                { name: 'description', type: 'string' }
+            ]
+        };
+        await modelsCollection.insertOne(productModelDef);
+
+        // 2. Insert initial document
+        const initialData = { name: 'No-Op Widget', price: 10, description: 'Initial.' };
+        const insertResult = await insertData(modelName, initialData, {}, testUser);
+        const docId = new ObjectId(insertResult.insertedIds[0]);
+
+        // 3. Edit ONLY a non-historized field
+        const updateData = { description: 'This change should not be recorded.' };
+        await editData(modelName, { _id: docId }, updateData, {}, testUser);
+
+        await sleep(2000);
+
+        // 4. Verify that NO new history record was created (only the 'create' record exists)
+        const historyCount = await historyCollection.countDocuments({ documentId: docId });
+        expect(historyCount).toBe(1);
+    });
+
+    it('should filter history records by date range', async () => {
+        // 1. Setup model
+        const modelName = generateUniqueName('productHistoryDateFilter');
+        const productModelDef = {
+            name: modelName,
+            description: "",
+            _user: testUser.username,
+            history: { enabled: true },
+            fields: [{ name: 'name', type: 'string' }]
+        };
+        await modelsCollection.insertOne(productModelDef);
+
+        // 2. Insert initial document
+        const insertResult = await insertData(modelName, { name: 'Time-traveling Widget' }, {}, testUser);
+        const docId = new ObjectId(insertResult.insertedIds[0]);
+
+        // 3. Create history entries at different times
+        // To simulate different timestamps, we'll manually insert history records
+        // as editData() would create them too close together in time.
+        await historyCollection.updateOne({ documentId: docId, version: 1 }, { $set: { timestamp: new Date('2023-01-10T10:00:00Z') } });
+
+        await historyCollection.insertOne({
+            documentId: docId, model: modelName, version: 2, operation: 'update',
+            timestamp: new Date('2023-02-15T12:00:00Z'), user: { username: testUser.username }, changes: { name: { from: 'v1', to: 'v2' } }
+        });
+        await historyCollection.insertOne({
+            documentId: docId, model: modelName, version: 3, operation: 'update',
+            timestamp: new Date('2023-02-20T14:00:00Z'), user: { username: testUser.username }, changes: { name: { from: 'v2', to: 'v3' } }
+        });
+        await historyCollection.insertOne({
+            documentId: docId, model: modelName, version: 4, operation: 'update',
+            timestamp: new Date('2023-03-05T16:00:00Z'), user: { username: testUser.username }, changes: { name: { from: 'v3', to: 'v4' } }
+        });
+
+        // Mock Express req/res objects
+        const mockReq = (query) => ({
+            params: { modelName, recordId: docId.toString() },
+            query,
+            me: testUser
+        });
+        const mockRes = () => {
+            const res = {};
+            res.status = (code) => { res.statusCode = code; return res; };
+            res.json = (data) => { res.body = data; return res; };
+            return res;
+        };
+
+        // 4. Test cases
+        // Case A: Filter for February
+        let req = mockReq({ startDate: '2023-02-01', endDate: '2023-02-28' });
+        let res = mockRes();
+        await handleGetHistoryRequest(req, res);
+        expect(res.body.success).toBe(true);
+        expect(res.body.count).toBe(2);
+        expect(res.body.data.map(d => d._v).sort()).toEqual([2, 3]);
+
+        // Case B: Filter starting from Feb 15th
+        req = mockReq({ startDate: '2023-02-15' });
+        res = mockRes();
+        await handleGetHistoryRequest(req, res);
+        expect(res.body.success).toBe(true);
+        expect(res.body.count).toBe(3); // v2, v3, v4
+
+        // Case C: Filter up to Feb 15th (inclusive)
+        req = mockReq({ endDate: '2023-02-15' });
+        res = mockRes();
+        await handleGetHistoryRequest(req, res);
+        expect(res.body.success).toBe(true);
+        expect(res.body.count).toBe(2); // v1, v2
+    });
 });