data-primals-engine 1.0.8 → 1.0.10

package/.github/workflows/node.js.yml

@@ -0,0 +1,59 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "develop", "main" ]
+  pull_request:
+    branches: [ "develop", "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    container:
+      image: node:18  # Specify the Docker version you needx
+      env:
+        NODE_OPTIONS: "--max_old_space_size=4096"
+        NODE_ENV: development
+      ports:
+        - 7635
+        - 27017
+    strategy:
+      matrix:
+        node-version: [18.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+    services:
+      mongodb:
+        image: mongo:5.0
+        ports:
+          - 27017:27017
+        options: >-
+          --health-cmd "mongo --eval 'db.serverStatus()'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install MongoDB Tools
+      run: |
+        wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add -
+        echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-5.0.list
+        apt-get update
+        apt-get install -y mongodb-org-tools
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '18.18.2'
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Run tests
+      run: npm test
+
+      

package/.github/workflows/npm-publish.yml

@@ -0,0 +1,23 @@
+name: Publish package to GitHub Packages
+on:
+  release:
+    types: [published]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      # Setup .npmrc file to publish to GitHub Packages
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+          registry-url: 'https://npm.pkg.github.com'
+          # Defaults to the user or organization that owns the workflow file
+          scope: '@octocat'
+      - run: npm ci
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/Dockerfile

@@ -0,0 +1,15 @@
+# Base image
+FROM node:18.18.2
+
+# Set working directory
+WORKDIR /
+
+# Copy package files and install dependencies
+COPY package.json ./
+RUN npm install
+
+# Copy application code
+COPY . .
+
+# Expose the application port
+EXPOSE 7633

package/DockerfileTest

@@ -0,0 +1,15 @@
+# Base image
+FROM node:18.18.2
+
+# Set working directory
+WORKDIR /
+
+# Copy package files and install dependencies
+COPY package.json ./
+RUN npm install
+
+# Copy application code
+COPY . .
+
+# Expose the application port
+EXPOSE 7635

package/README.md

@@ -194,6 +194,195 @@ curl -X DELETE http://localhost:7633/api/data?_user=demo \
          }'
 ```
 
+## Other operations
+
+Make sure you use the code below to initialize the user : 
+```javascript
+import { Engine } from 'data-primals-engine/engine';
+import { insertData, searchData } from 'data-primals-engine/modules/data';
+
+// Ensure the engine is initialized
+const engine = await Engine.Create();
+const currentUser = await engine.userProvider.findUserByUsername('demo');
+if (!currentUser) {
+    throw new Error("Could not retrieve the user. Please check credentials or user provider.");
+}
+console.log(`Successfully authenticated as ${currentUser.username}`);
+```
+
+### insertData(modelName, data, files, user)
+
+> Inserts one or more documents, intelligently handling nested relationships.
+
+```javascript
+// Uses the `currentUser` object defined above
+const newProduct = { name: 'Super Widget', price: 99.99, status: 'available' };
+const result = await insertData('product', newProduct, {}, currentUser);
+
+if (result.success) {
+    console.log(`Successfully inserted document with ID: ${result.insertedIds[0]}`);
+}
+```
+
+### editData(modelName, filter, data, files, user)
+
+> Updates existing data matching the filter.
+
+Example:
+
+```javascript
+await editData(
+    "userProfile",
+    { _id: "507f1f77bcf86cd799439011" },
+    { bio: "Updated bio text" },
+    null, // No files
+    currentUser
+);
+```
+
+### patchData(modelName, filter, data, files, user)
+
+> Partially updates data (only modifies specified fields).
+
+Example:
+
+```javascript
+await patchData(
+    "settings",
+    { userId: "507f1f77bcf86cd799439011" },
+    { theme: "dark" },
+    null,
+    currentUser
+);
+```
+
+### deleteData(modelName, ids, filter, user)
+
+>Deletes data with cascading relation cleanup.
+
+Examples:
+
+```javascript
+// Delete by IDs
+await deleteData("comments", ["61d1f1a9e3f1a9e3f1a9e3f1"], null, user);
+
+// Delete by filter
+await deleteData("logs", null, { createdAt: { $lt: "2023-01-01" } }, user);
+```
+
+### searchData({user, query})
+
+Powerful search with relation expansion and filtering.
+
+Query Options:
+
+- model: Model name to search
+- filter: MongoDB-style filter
+- depth: Relation expansion depth (default: 1)
+- limit/page: Pagination
+- sort: Sorting criteria
+
+Example:
+```javascript
+const results = await searchData({
+    user: currentUser,
+    query: {
+        model: "blogPost",
+        filter: { status: "published" },
+        depth: 2, // Expand author and comments
+        limit: 10,
+        sort: "createdAt:DESC"
+    }
+});
+```
+
+## Import/Export
+### importData(options, files, user)
+> Imports data from JSON/CSV files.
+
+Supported Formats:
+
+- JSON arrays
+- JSON with model-keyed objects
+- CSV with headers or field mapping
+
+Example:
+
+```javascript
+const result = await importData(
+    {
+        model: "products",
+        hasHeaders: true
+    },
+    { 
+        file: req.files.myFileField // from multipart body  
+    },
+    currentUser
+);
+```
+
+### exportData(options, user)
+> Exports data to a structured format.
+
+Example:
+
+```javascript
+await exportData(
+    {
+        models: ["products", "categories"],
+        depth: 1, // Include relations
+        lang: "fr" // Localized data
+    },
+    currentUser
+);
+// Returns: { success: true, data: { products: [...], categories: [...] } }
+```
+
+## Backup & Restore
+### dumpUserData(user)
+> Creates an encrypted backup of user data.
+
+Features:
+
+- Automatic encryption
+- S3 or local storage
+- Retention policies by plan (daily/weekly/monthly)
+
+Example:
+
+```javascript
+await dumpUserData(currentUser);
+// Backup saved to S3 or ./backups/
+```
+
+### loadFromDump(user, options)
+
+> Restores user data from backup.
+
+Options:
+- modelsOnly: Restore only model definitions
+
+Example:
+
+```javascript
+await loadFromDump(currentUser, { modelsOnly: false });
+// Full restore including data
+```
+
+## Pack Management
+
+### installPack(logger, packId, user, lang)
+
+> Installs a predefined data pack.
+
+Example:
+
+```javascript
+const result = await installPack(logger, "61d1f1a9e3f1a9e3f1a9e3f1", user, "en");
+// Returns installation summary
+```
+
+
 ---
 
 ## 📁 Project Structure
@@ -212,6 +401,34 @@ data-primals-engine/
 └── server.js
 ```
 
+## Workflows: Automate Your Business Logic
+
+> Workflows are the automation engine of your application. 
+
+They allow you to define complex business processes that run in response to specific events, without writing custom code. 
+
+This is perfect for tasks like **sending welcome emails**, managing **order fulfillment**, or triggering data synchronization.
+
+A workflow is composed of two main parts: **Triggers** and **Actions**.
+
+> A 'workflowTrigger' is the event that initiates a workflow run.
+- **DataAdded**: Fires when a new document is created (e.g., a new user signs up).
+- **DataEdited**: Fires when a document is updated (e.g., an order status changes to "shipped").
+- **DataDeleted**: Fires when a document is removed.
+- **Scheduled**: Runs at a specific time or interval using a Cron expression (e.g., 0 0 * * * for a nightly data cleanup job).
+- **Manual**: Triggered on-demand via an API call, allowing you to integrate workflows into any part of your application.
+
+> A 'workflowAction' is the individual steps a workflow executes. You can chain them together to create sophisticated logic.
+- **CreateData**: Create a new document in any model.
+- **UpdateData**: Modify one or more existing documents that match a filter.
+- **SendEmail**: Send a transactional email using dynamic data from the trigger.
+- **CallWebhook**: Make an HTTP request (GET, POST, etc.) to an external service or API.
+- **ExecuteScript**: Run a custom JavaScript snippet for complex logic, data transformation, or conditional branching.
+- **GenerateAIContent**: Use an integrated AI provider (like OpenAI or Gemini) to generate text, summarize content, or make decisions.
+- **Wait**: Pause the workflow for a specific duration before continuing to the next step
+
+See the details of the workflow models for more details.
+
 ---
 
 ## 🤝 Contributing

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "data-primals-engine",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "data-primals-engine is the package responsible from handling large amount of data in a practical and performant way. It can handle large amount of data in a practical and performant way. It can also get workflow models working (for automation), and fully supports internationalisation.",
   "main": "src/engine.js",
   "type": "module",
   "scripts": {
+    "test": "cross-env MONGO_DB_URL=\"mongodb://localhost:27017\" PORT=7635 vitest --no-file-parallelism",
     "preinstall": "npx force-resolutions",
     "devserver": "cross-env NODE_ENV=development node server.js",
     "server": "cross-env NODE_ENV=production node server.js",
@@ -39,11 +40,9 @@
     "chalk": "^5.4.1",
     "check-disk-space": "^3.4.0",
     "compression": "^1.8.0",
-    "connect-mongo": "^5.1.0",
     "cookie-parser": "^1.4.7",
     "cronstrue": "^3.1.0",
     "csv-parse": "^5.6.0",
-    "csv-parser": "^3.2.0",
     "date-fns": "^4.1.0",
     "express": "^4.21.2",
     "express-csrf-double-submit-cookie": "^1.2.1",
@@ -61,7 +60,6 @@
     "nodemailer": "^6.10.0",
     "openai": "^5.8.2",
     "randomcolor": "^0.6.2",
-    "rate-limiter-flexible": "^5.0.5",
     "react-helmet": "^6.1.0",
     "react-i18next": "^15.4.1",
     "react-markdown": "^10.1.0",
@@ -70,10 +68,9 @@
     "sirv": "^3.0.1",
     "swagger-ui-express": "^5.0.1",
     "tar": "^7.4.3",
-    "tar-fs": "^3.0.8",
     "uniqid": "^5.4.0",
-    "yaml": "^2.8.0",
-    "zlib": "^1.0.5"
+    "vitest": "^3.2.3",
+    "yaml": "^2.8.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.17.0",

package/src/engine.js

@@ -17,7 +17,7 @@ import formidableMiddleware from 'express-formidable';
 const isProduction = process.env.NODE_ENV === 'production'
 
 // Connection URL
-export const dbUrl = process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017';
+export const dbUrl = process.env.CI ? 'mongodb://mongodb:27017' : (process.env.MONGO_DB_URL || 'mongodb://127.0.0.1:27017');
 export const MongoClient = new InternalMongoClient(dbUrl, { maxPoolSize: 20 });
 
 // Database Name
@@ -87,7 +87,7 @@ export const Engine = {
             }
         };
 
-        await Promise.all(Config.Get('modules').map(async module => {
+        await Promise.all(Config.Get('modules', []).map(async module => {
             try {
                 if( fs.existsSync(module)){
                     return await importModule(module);