data-primals-engine 1.2.1 → 1.2.3

package/src/packs.js

@@ -39,6 +39,14 @@ export const getAllPacks = async () => {
         return perms;
     }
 
+    const envSmtp = [
+        { "name": "SMTP_HOST", "value": "smtp.example.com", "description": "SMTP server host for sending emails." },
+        { "name": "SMTP_PORT", "value": "587", "description": "SMTP server port." },
+        { "name": "SMTP_USER", "value": "user@example.com", "description": "Username for SMTP authentication." },
+        { "name": "SMTP_PASS", "value": "your_smtp_password", "description": "Password for SMTP authentication." },
+        { "name": "SMTP_FROM", "value": "\"My Store\" <noreply@example.com>", "description": "The 'From' address for outgoing emails." }
+    ];
+
     const categories = [ 'News', 'Blog', 'Products', 'Services', 'Store', 'Events', 'Forums', 'Contact', 'Support'];
     const tags = [ 'info', 'incident', 'maintenance', 'feature', 'hint', 'bugfix', 'question'];
 
@@ -48,6 +56,242 @@ export const getAllPacks = async () => {
         {name: 'visitor', perms: ['API_SEARCH_DATA_webpage','API_SEARCH_DATA_content', 'API_SEARCH_DATA_lang', 'API_SEARCH_DATA_currency', 'API_SEARCH_DATA_taxonomy']}];
 
     return [
+        {
+            "name": "Marketing & Campaigning",
+            "description": "Launch powerful, personalized, and scalable email campaigns. This pack uses dynamic audiences and a robust workflow to send emails in chunks, ensuring high performance. Depends on the 'Customer Relationship Management (CRM)' pack.",
+            "tags": ["marketing", "email", "campaign", "workflow"],
+            "models": ["env", "contact", "workflow", "workflowStep", "workflowAction", "workflowRun", "workflowTrigger",
+                {
+                    "name": "campaign",
+                    "description": "Defines an email marketing campaign.",
+                    "fields": [
+                        { "name": "name", "type": "string", "required": true, "asMain": true },
+                        { "name": "subject", "type": "string", "required": true },
+                        { "name": "content", "type": "richtext", "required": true },
+                        { "name": "status", "type": "enum", "items": ["draft", "scheduled", "in_progress", "finished", "cancelled"], "default": "draft" },
+                        { "name": "audience", "type": "relation", "relation": "audience" },
+                        { "name": "processedRecipients", "type": "array", "itemsType": "string", "hint": "List of processed contact IDs." }
+                    ]
+                },
+                {
+                    "name": "audience",
+                    "description": "Defines a dynamic segment of contacts based on a filter.",
+                    "fields": [
+                        { "name": "name", "type": "string", "required": true, "asMain": true },
+                        { "name": "description", "type": "string" },
+                        { "name": "filter", "type": "code", "language": "json", "required": true, "conditionBuilder": true, "hint": "A MongoDB filter to select contacts. E.g., { \"tags\": \"newsletter\" }" }
+                    ]
+                }
+            ],
+            "data":{
+                "all": {
+                    "audience": [
+                        {
+                            "name": "Example Audience for Campaigning",
+                            "description": "An example audience targeting specific contacts from the CRM pack.",
+                            "filter": { "$or": [{ "$eq": ["$email", "alice.martin@innovatech.com"] }, { "$eq": ["$email", "bob.durand@globalexports.com"] }] }
+                        }
+                    ],
+                    "campaign": [
+                        {
+                            "name": "Q3 Product Launch",
+                            "subject": "🚀 Discover Our New Products!",
+                            "content": "<h1>Hello {recipient.firstName},</h1><p>We are excited to introduce our latest product line. We think you'll love it!</p><p>Best regards,<br>The Team</p>",
+                            "status": "draft",
+                            "audience": { "$link": { "name": "Example Audience for Campaigning", "_model": "audience" } }
+                        }
+                    ],
+                    "workflow": [{
+                        "name": "Campaign Emailing Workflow",
+                        "description": "A scalable workflow that sends campaign emails in chunks by dynamically querying contacts from an audience.",
+                        "startStep": { "$link": { "name": "Start Campaign Processing", "_model": "workflowStep" } }
+                    }],
+                    "workflowAction": [
+                        {
+                            "name": "Set Campaign to 'in_progress'",
+                            "type": "UpdateData",
+                            "targetModel": "campaign",
+                            "targetSelector": { "_id": "{triggerData._id}" },
+                            "fieldsToUpdate": { "status": "in_progress" }
+                        },
+                        {
+                            "name": "Get Next Recipient Chunk",
+                            "type": "ExecuteScript",
+                            "script": `
+const chunkSize = 10; // Process 10 recipients per run
+const campaign = context.triggerData;
+const audience = await db.findOne("audience",{"_id": campaign.audience});
+
+if (!audience || !audience.filter) {
+logger.error('Campaign audience or audience filter is not defined.');
+return { chunk: [], message: 'Campaign audience or audience filter is not defined.'}; // Returning an empty chunk will stop the workflow.
+}
+
+const processedIds = campaign.processedRecipients || [];
+
+const query = {
+    '$and': [
+        audience.filter,
+        { '$not':{ '$in': ["$_id", processedIds] } }
+    ]
+};
+
+logger.info('Finding next chunk with filter:', JSON.stringify(query));
+
+const searchResult = await db.find('contact', query, { limit: chunkSize });
+const chunk = searchResult.data || [];
+
+logger.info(\`Found \${chunk.length} recipients for the next chunk.\`);
+
+return { chunk }; // This chunk is passed to the next action via context.result
+`
+                        },
+                        {
+                            "name": "Send Email to Chunk",
+                            "type": "SendEmail",
+                            "emailRecipients": ["{context.result.chunk}"],
+                            "emailSubject": "{triggerData.subject}",
+                            "emailContent": "{triggerData.content}"
+                        },
+                        {
+                            "name": "Update Processed Recipients",
+                            "type": "ExecuteScript",
+                            "script": `
+const campaignId = context.triggerData._id;
+const emailResult = context.emailResult; 
+
+if (!emailResult || !Array.isArray(emailResult.sent) || emailResult.sent.length === 0) {
+    logger.info('No recipients were successfully sent an email in this chunk.');
+    // Return the original chunk from the first script to allow the condition to check it
+    return { processedChunk: context.result.chunk };
+}
+
+const processedIds = emailResult.sent.map(recipient => recipient._id.toString());
+
+logger.info(\`Updating campaign \${campaignId} with \${processedIds.length} new processed IDs.\`);
+
+await db.update(
+    'campaign',
+    { _id: campaignId },
+    { 'processedRecipients': [...campaign.processedRecipients, ...processedIds] }
+);
+
+// Return the original chunk for the condition check step
+return { processedChunk: context.result.chunk };
+`
+                        },
+                        {
+                            "name": "Set Campaign to 'finished'",
+                            "type": "UpdateData",
+                            "targetModel": "campaign",
+                            "targetSelector": { "_id": "{triggerData._id}" },
+                            "fieldsToUpdate": { "status": "finished" }
+                        }
+                    ],
+                    "workflowStep": [
+                        {
+                            "name": "Start Campaign Processing",
+                            "workflow": { "$link": { "name": "Campaign Emailing Workflow", "_model": "workflow" } },
+                            "actions": { "$link": { "name": "Set Campaign to 'in_progress'", "_model": "workflowAction" } },
+                            "onSuccessStep": { "$link": { "name": "Process Recipient Chunk", "_model": "workflowStep" } }
+                        },
+                        {
+                            "name": "Process Recipient Chunk",
+                            "workflow": { "$link": { "name": "Campaign Emailing Workflow", "_model": "workflow" } },
+                            "actions":
+                                { "$link": { "$or": [
+                                    {"$eq": ["$name", "Get Next Recipient Chunk"]},
+                                    {"$eq": ["$name", "Send Email to Chunk"]},
+                                    {"$eq": ["$name", "Update Processed Recipients"]}
+                                ],"_model": "workflowAction"}},
+                            "onSuccessStep": { "$link": { "name": "Check if Campaign is Complete", "_model": "workflowStep" } }
+                        },
+                        {
+                            "name": "Check if Campaign is Complete",
+                            "workflow": { "$link": { "name": "Campaign Emailing Workflow", "_model": "workflow" } },
+                            "conditions": { "$gt": [{ "$size": {$ifNull:["{context.result.processedChunk}", []]} }, 0] },
+                            "onSuccessStep": { "$link": { "name": "Process Recipient Chunk", "_model": "workflowStep" } },
+                            "onFailureStep": { "$link": { "name": "Finish Campaign", "_model": "workflowStep" } }
+                        },
+                        {
+                            "name": "Finish Campaign",
+                            "workflow": { "$link": { "name": "Campaign Emailing Workflow", "_model": "workflow" } },
+                            "actions": { "$link": { "name": "Set Campaign to 'finished'", "_model": "workflowAction" } },
+                            "isTerminal": true
+                        }
+                    ],
+                    "workflowTrigger": [{
+                        "name": "On Campaign Scheduled",
+                        "workflow": { "$link": { "name": "Campaign Emailing Workflow", "_model": "workflow" } },
+                        "type": "manual",
+                        "onEvent": "DataEdited",
+                        "targetModel": "campaign",
+                        "dataFilter": { "$eq": ["$status", "scheduled"] },
+                        "isActive": true
+                    }],
+                    "env": envSmtp
+                }
+            }
+        },
+        /*
+        {
+            "name": "Social Media Publisher",
+            "description": "Automate your social media presence. This pack provides a workflow to automatically post new blog articles to Twitter and LinkedIn. Requires the 'Website Starter Pack'.",
+            "tags": ["social media", "marketing", "automation", "workflow"],
+            "dependencies": ["Website Starter Pack"],
+            "models": ["workflow", "workflowStep", "workflowAction", "workflowTrigger", "env"],
+            "data": {
+                "all": {
+                    "workflow": [{
+                        "name": "Publish New Blog Post to Socials",
+                        "description": "Automatically posts a link to a new blog post on Twitter and LinkedIn.",
+                        "startStep": {"$link": {"name": "Post on Social Networks", "_model": "workflowStep"}}
+                    }],
+                    "workflowAction": [
+                        {
+                            "name": "Post to Twitter",
+                            "type": "PostToSocialMedia",
+                            "provider": "Twitter",
+                            "content": "📰 New blog post published: {triggerData.title}! Check it out here: https://your-website.com/blog/{triggerData.slug}"
+                        },
+                        {
+                            "name": "Post to LinkedIn",
+                            "type": "PostToSocialMedia",
+                            "provider": "LinkedIn",
+                            "content": "We've just published a new article: '{triggerData.title}'.\n\n{triggerData.summary}\n\nRead the full post on our blog: https://your-website.com/blog/{triggerData.slug}\n#YourIndustry #BlogPost"
+                        }
+                    ],
+                    "workflowStep": [{
+                        "name": "Post on Social Networks",
+                        "workflow": {"$link": {"name": "Publish New Blog Post to Socials", "_model": "workflow"}},
+                        "actions": {
+                            "$link": {
+                                "$or": [
+                                    {"$eq": ["$name", "Post to Twitter"]},
+                                    {"$eq": ["$name", "Post to LinkedIn"]}
+                                ],
+                                "_model": "workflowAction"
+                            }
+                        },
+                        "isTerminal": true
+                    }],
+                    "workflowTrigger": [{
+                        "name": "On New Blog Post Added",
+                        "workflow": {"$link": {"name": "Publish New Blog Post to Socials", "_model": "workflow"}},
+                        "type": "manual",
+                        "onEvent": "DataAdded",
+                        "targetModel": "content",
+                        "dataFilter": {"category": {"$find": {"name": "Blog"}}},
+                        "isActive": true
+                    }],
+                    "env": [
+                        {"name": "TWITTER_API_KEY", "value": "your_key_here"},
+                        {"name": "TWITTER_API_SECRET", "value": "your_secret_here"},
+                        {"name": "LINKEDIN_ACCESS_TOKEN", "value": "your_token_here"}
+                    ]
+                }
+            }
+        },*/
         {
             "name": "E-commerce Starter Kit",
             "description": "Launch your online store in just a few clicks. This pack includes templates for products, orders, and customers, as well as sample data, KPIs, alerts and an order fulfillment workflow (with sending email).",
@@ -55,13 +299,7 @@ export const getAllPacks = async () => {
             "models": ["env", "taxonomy", "product", "productVariant", "brand", "currency", "order", "shipment", "review", "cart", "cartItem", "discount", "workflow", "workflowStep", "workflowAction","workflowRun", "workflowTrigger", "translation", "lang", "kpi", "dashboard", "alert", "return"],
             "data": {
                 "all": {
-                    "env": [
-                        { "name": "SMTP_HOST", "value": "smtp.example.com", "description": "SMTP server host for sending emails." },
-                        { "name": "SMTP_PORT", "value": "587", "description": "SMTP server port." },
-                        { "name": "SMTP_USER", "value": "user@example.com", "description": "Username for SMTP authentication." },
-                        { "name": "SMTP_PASS", "value": "your_smtp_password", "description": "Password for SMTP authentication." },
-                        { "name": "SMTP_FROM", "value": "\"My Store\" <noreply@example.com>", "description": "The 'From' address for outgoing emails." }
-                    ],
+                    "env":envSmtp,
                     "taxonomy": [
                         { "name": "E-commerce", "type": "category" },
                         { "name": "Clothes", "type": "category", "parent": { "$find": { "name": "E-commerce" } } },

package/test/data.backup.integration.test.js

@@ -14,13 +14,14 @@ import {
 
 import {
     modelsCollection as getAppModelsCollection,
-    getCollectionForUser as getAppUserCollection
+    getCollectionForUser as getAppUserCollection, getCollectionForUser
 } from 'data-primals-engine/modules/mongodb';
 import process from "node:process";
 
 import { dumpUserData, loadFromDump } from 'data-primals-engine/modules/data';
 import fs from "node:fs";
 import {initEngine} from "../src/setenv.js";
+import {getUserCollectionName} from "../src/modules/mongodb.js";
 
 vi.mock('data-primals-engine/engine', async(importOriginal) => {
     const mod = await importOriginal() // type is inferred
@@ -70,7 +71,7 @@ beforeAll(async () => {
     fs.readdirSync(backupDir).forEach(file => {
         fs.unlinkSync(path.join(backupDir, file));
     });
-    vi.stubEnv('S3_CONFIG_ENCRYPTION_KEY', '00000000000000000000000000000000');
+    vi.stubEnv('ENCRYPTION_KEY', '00000000000000000000000000000000');
     vi.stubEnv('OPENAI_API_KEY', '00000000000000000000000000000000');
     // You might need to create a model first if your dumpUserData requires it
     await createModel(testModelDefinition);
@@ -88,6 +89,8 @@ afterAll(async () => {
         });
         fs.rmdirSync(backupDir); // Remove the directory itself
     }
+    const coll = await getCollectionForUser(mockUser);
+    await coll.drop()
 });
 
 beforeAll(async () =>{

package/test/data.integration.test.js

@@ -1,22 +1,25 @@
 // __tests__/data.integration.test.js
 import { ObjectId } from 'mongodb';
-import {expect, describe, it, beforeEach, beforeAll, afterAll} from 'vitest';
+import {vi, expect, describe, it, beforeEach, beforeAll, afterAll} from 'vitest';
 import { Config } from '../src/config.js';
-
 import {
     insertData,
     editData,
     deleteData,
-    searchData, installPack, deleteModels, createModel
+    searchData, installPack, deleteModels, createModel, patchData
 } from 'data-primals-engine/modules/data';
 
 import {
     modelsCollection as getAppModelsCollection,
     getCollection,
-    getCollectionForUser as getAppUserCollection
+    getCollectionForUser as getAppUserCollection, getCollectionForUser
 } from 'data-primals-engine/modules/mongodb';
 import {getRandom} from "../src/core.js";
 import {generateUniqueName, initEngine} from "../src/setenv.js";
+import {processWorkflowRun} from "data-primals-engine/modules/workflow";
+import {sendEmail} from "../src/email.js";
+import {MongoDatabase} from "../src/engine.js";
+import {getUserCollectionName} from "../src/modules/mongodb.js";
 
 let testModelsColInstance;
 let testDatasColInstance;
@@ -828,6 +831,8 @@ describe('Intégration des fonctions CRUD de données avec validation complète'
             await deleteModels({ name: orderModel.name, _user: user.username });
             await deleteData(productModel.name, {}, user);
             await deleteData(orderModel.name, {}, user);
+            const coll = await getCollectionForUser(user);
+            await coll.drop();
         });
 
         it('should ALLOW inserting data with a valid relation', async () => {
@@ -871,4 +876,5 @@ describe('Intégration des fonctions CRUD de données avec validation complète'
             await deleteData(orderModel.name, initialOrder.insertedIds, user);
         });
     });
+
 });

package/test/events.test.js

@@ -0,0 +1,202 @@
+// events.test.js
+import { Event } from '../src/events.js';
+import {vitest} from "vitest";
+import {expect, describe, it, beforeEach, beforeAll, afterAll} from 'vitest';
+
+describe('Event System', () => {
+    beforeEach(() => {
+        // Clear all events before each test to ensure a clean state
+        for (const system in Event) {
+            if (Event.hasOwnProperty(system) && typeof Event[system] === 'object') {
+                for (const eventName in Event[system]) {
+                    if (Event[system].hasOwnProperty(eventName)) {
+                        delete Event[system][eventName];
+                    }
+                }
+            }
+        }
+    });
+
+    it('should allow listening for and triggering an event', () => {
+        const mockCallback = vitest.fn();
+        const eventName = 'testEvent';
+
+        Event.Listen(eventName, mockCallback);
+        Event.Trigger(eventName);
+
+        expect(mockCallback).toHaveBeenCalledTimes(1);
+    });
+    it('should pass arguments to the event callback', () => {
+        const mockCallback = vitest.fn();
+        const eventName = 'testEventWithArgs';
+        const arg1 = 'hello';
+        const arg2 = 123;
+
+        Event.Listen(eventName, mockCallback, "priority", "medium");
+        Event.Trigger(eventName, "priority", "medium", arg1, arg2);
+
+        expect(mockCallback).toHaveBeenCalledTimes(1);
+        expect(mockCallback).toHaveBeenCalledWith(arg1, arg2);
+    });
+
+    it('should allow removing a specific callback', () => {
+        const mockCallback1 = vitest.fn();
+        const mockCallback2 = vitest.fn();
+        const eventName = 'testEventRemoveCallback';
+
+        Event.Listen(eventName, mockCallback1);
+        Event.Listen(eventName, mockCallback2);
+        Event.RemoveCallback(eventName, mockCallback1);
+        Event.Trigger(eventName);
+
+        expect(mockCallback1).not.toHaveBeenCalled();
+        expect(mockCallback2).toHaveBeenCalledTimes(1);
+    });
+
+    it('should not trigger a callback if the event is not listened to', () => {
+        const mockCallback = vitest.fn();
+        const eventName = 'nonExistentEvent';
+
+        Event.Trigger(eventName);
+
+        expect(mockCallback).not.toHaveBeenCalled();
+    });
+
+    it('should handle different systems and layers', () => {
+        const mockCallbackPriority = vitest.fn();
+        const mockCallbackLog = vitest.fn();
+        const eventName = 'testEventMultiSystem';
+
+        Event.Listen(eventName, mockCallbackPriority, 'priority', 'high');
+        Event.Listen(eventName, mockCallbackLog, 'log', 'info');
+
+        Event.Trigger(eventName, 'priority', 'high');
+        expect(mockCallbackPriority).toHaveBeenCalledTimes(1);
+        expect(mockCallbackLog).not.toHaveBeenCalled();
+
+        vitest.clearAllMocks(); // Reset mock counts
+
+        Event.Trigger(eventName, 'log', 'info');
+        expect(mockCallbackPriority).not.toHaveBeenCalled();
+        expect(mockCallbackLog).toHaveBeenCalledTimes(1);
+    });
+
+    it('should throw an error if an invalid system or layer is used with Listen', () => {
+        expect(() => {
+            Event.Listen('invalidEvent', () => {}, 'invalidSystem', 'invalidLayer');
+        }).toThrowError(/System 'invalidSystem' does not exist./);
+
+        expect(() => {
+            Event.Listen('invalidEvent', () => {}, 'priority', 'invalidLayer');
+        }).toThrowError(/Layer 'invalidLayer' does not exist/);
+    });
+
+    describe('Event.Trigger result merging', () => {
+        it('should merge objects from multiple callbacks using spread operator', () => {
+            const eventName = 'mergeObjectEvent';
+            const callback1 = vitest.fn().mockReturnValue({ a: 1, b: 2 });
+            const callback2 = vitest.fn().mockReturnValue({ b: 3, c: 4 }); // b will be overwritten
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+
+            const result = Event.Trigger(eventName);
+
+            expect(result).toEqual({ a: 1, b: 3, c: 4 });
+        });
+
+        it('should concatenate arrays from multiple callbacks', () => {
+            const eventName = 'mergeArrayEvent';
+            const callback1 = vitest.fn().mockReturnValue([1, 2]);
+            const callback2 = vitest.fn().mockReturnValue([3, 4]);
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+
+            const result = Event.Trigger(eventName);
+
+            expect(result).toEqual([1, 2, 3, 4]);
+        });
+
+        it('should concatenate strings from multiple callbacks', () => {
+            const eventName = 'mergeStringEvent';
+            const callback1 = vitest.fn().mockReturnValue('Hello ');
+            const callback2 = vitest.fn().mockReturnValue('World');
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+
+            const result = Event.Trigger(eventName);
+
+            expect(result).toBe('Hello World');
+        });
+
+        it('should add numbers from multiple callbacks', () => {
+            const eventName = 'mergeNumberEvent';
+            const callback1 = vitest.fn().mockReturnValue(10);
+            const callback2 = vitest.fn().mockReturnValue(20);
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+
+            const result = Event.Trigger(eventName);
+
+            expect(result).toBe(30);
+        });
+
+        it('should perform a logical AND on booleans from multiple callbacks', () => {
+            const eventName = 'mergeBooleanEvent';
+
+            // Case 1: true && true -> true
+            const cb_true1 = vitest.fn().mockReturnValue(true);
+            const cb_true2 = vitest.fn().mockReturnValue(true);
+            Event.Listen(eventName, cb_true1);
+            Event.Listen(eventName, cb_true2);
+            expect(Event.Trigger(eventName)).toBe(true);
+            Event.RemoveAllCallbacks(eventName); // Clean up for next case
+
+            // Case 2: true && false -> false
+            const cb_false1 = vitest.fn().mockReturnValue(false);
+            Event.Listen(eventName, cb_true1);
+            Event.Listen(eventName, cb_false1);
+            expect(Event.Trigger(eventName)).toBe(false);
+            Event.RemoveAllCallbacks(eventName);
+
+            // Case 3: false && true -> false
+            Event.Listen(eventName, cb_false1);
+            Event.Listen(eventName, cb_true1);
+            expect(Event.Trigger(eventName)).toBe(false);
+        });
+
+        it('should handle mixed-type return values', () => {
+            const eventName = 'mergeMixedEvent';
+            const callback1 = vitest.fn().mockReturnValue(100); // number
+            const callback2 = vitest.fn().mockReturnValue({ a: 1 }); // object
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+
+            const result = Event.Trigger(eventName);
+
+            // The logic initializes `ret` with the first value (100).
+            // When the second callback returns an object, it re-initializes `ret` to {}
+            // and merges the object. The final result is the object.
+            expect(result).toEqual({ a: 1 });
+        });
+
+        it('should ignore null and undefined return values while merging', () => {
+            const eventName = 'ignoreNullsEvent';
+            const callback1 = vitest.fn().mockReturnValue(null);
+            const callback2 = vitest.fn().mockReturnValue({ data: 'important' });
+            const callback3 = vitest.fn().mockReturnValue(undefined);
+
+            Event.Listen(eventName, callback1);
+            Event.Listen(eventName, callback2);
+            Event.Listen(eventName, callback3);
+
+            const result = Event.Trigger(eventName);
+
+            expect(result).toEqual({ data: 'important' });
+        });
+    });
+});