data-primals-engine 1.7.1 → 1.7.2

package/src/modules/assistant/constants.js

@@ -1,17 +1,24 @@
-
-export const providers = {
-    "OpenAI" : {
-        key:"OPENAI_API_KEY",
-        defaultModel: 'gpt-3.5-turbo'
-    },
-    "Google": {
-        key:"GOOGLE_API_KEY"
-    },
-    "DeepSeek": {
-        key: "DEEPSEEK_API_KEY",
-        defaultModel: 'deepseek-chat'
-    },
-    "Anthropic": {
-        key:"ANTHROPIC_API_KEY"
-    }
+
+export const providers = {
+    "OpenAI" : {
+        key:"OPENAI_API_KEY",
+        defaultModel: 'gpt-4o-mini',
+        generationModel: 'gpt-4o-mini' // Excellent for JSON generation
+    },
+    "Google": {
+        key:"GOOGLE_API_KEY",
+        defaultModel: 'gemini-1.5-flash',
+        generationModel: 'gemini-1.5-flash'
+    },
+    "DeepSeek": {
+        key: "DEEPSEEK_API_KEY",
+        defaultModel: 'deepseek-chat',
+        // As per the error message, 'deepseek-v2-coder' is a good candidate for code/JSON generation
+        generationModel: 'deepseek-coder'
+    },
+    "Anthropic": {
+        key:"ANTHROPIC_API_KEY",
+        defaultModel: 'claude-3-haiku-20240307',
+        generationModel: 'claude-3-haiku-20240307' // Haiku is fast and good with JSON
+    }
 };

package/src/modules/assistant/providers.js

@@ -1,38 +1,78 @@
-import { Config } from "../../config.js";
-import { assistantConfig } from "../../constants.js";
-import { Logger } from "../../gameObject.js";
-
-const logger = new Logger("AssistantProviders");
-
-export const getAIProvider = async (aiProvider, aiModel, apiKey) => {
-    const maxTokens = Config.Get('assistant.maxTokens', assistantConfig.maxTokens);
-    try {
-        switch (aiProvider) {
-            case 'OpenAI': {
-                const { ChatOpenAI } = await import("@langchain/openai");
-                return new ChatOpenAI({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
-            }
-            case 'Google': {
-                const { ChatGoogleGenerativeAI } = await import("@langchain/google-genai");
-                return new ChatGoogleGenerativeAI({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
-            }
-            case 'DeepSeek': {
-                const { ChatDeepSeek } = await import("@langchain/deepseek");
-                return new ChatDeepSeek({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
-            }
-            case 'Anthropic': {
-                const { ChatAnthropic } = await import("@langchain/anthropic");
-                return new ChatAnthropic({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
-            }
-            default:
-                throw new Error(`Unsupported AI provider: ${aiProvider}`);
-        }
-    } catch (e) {
-        if (e.code === 'ERR_MODULE_NOT_FOUND') {
-            logger.error(`[Assistant] The package for the '${aiProvider}' provider is not installed. Please run 'npm install @langchain/${aiProvider.toLowerCase()}' to use this provider.`);
-            throw new Error(`The AI provider '${aiProvider}' is not installed. Please ask the administrator to install the corresponding package.`);
-        }
-        logger.error(`[Assistant] Error initializing AI provider '${aiProvider}': ${e.message}`);
-        throw e; // Re-throw other errors
-    }
+import { Config } from "../../config.js";
+import {assistantConfig} from "../../constants.js";
+import { Logger } from "../../gameObject.js";
+import { getCollectionForUser } from "../mongodb.js";
+import { providers } from "./constants.js";
+import process from "node:process";
+
+const logger = new Logger("AssistantProviders");
+
+/**
+ * Finds the first available AI provider based on configured API keys for a given user.
+ * It checks the user's specific environment variables first, then falls back to the machine's process.env.
+ * @param {object} user - The user object.
+ * @param {string|null} [preferredProvider=null] - If specified, will only check for this provider.
+ * @returns {Promise<{provider: string, apiKey: string}|null>} An object with the provider name and API key, or null if none are found.
+ */
+export async function findFirstAvailableProvider(user, preferredProvider = null) {
+    const envCollection = await getCollectionForUser(user);
+    const providerNames = preferredProvider ? [preferredProvider] : Object.keys(providers);
+
+    for (const pName of providerNames) {
+        const providerInfo = providers[pName];
+        if (!providerInfo) continue;
+
+        const envKeyName = providerInfo.key;
+        const userEnvVar = await envCollection.findOne({ _model: 'env', name: envKeyName, _user: user.username });
+        const key = userEnvVar?.value || process.env[envKeyName];
+
+        if (key) {
+            return { provider: pName, apiKey: key };
+        }
+    }
+    return null;
+}
+
+export const getAIProvider = async (aiProvider, aiModel, apiKey, isJsonMode = false) => {
+    const maxTokens = Config.Get('assistant.maxTokens', assistantConfig.maxTokens);
+    const modelOptions = {
+        apiKey,
+        model: aiModel,
+        temperature: 0.7,
+        maxTokens
+    };
+
+    if (isJsonMode) {
+        modelOptions.response_format = { type: "json_object" };
+    }
+
+    try {
+        switch (aiProvider) {
+            case 'OpenAI': {
+                const { ChatOpenAI } = await import("@langchain/openai");
+                return new ChatOpenAI(modelOptions);
+            }
+            case 'Google': {
+                const { ChatGoogleGenerativeAI } = await import("@langchain/google-genai");
+                return new ChatGoogleGenerativeAI(modelOptions);
+            }
+            case 'DeepSeek': {
+                const { ChatDeepSeek } = await import("@langchain/deepseek");
+                return new ChatDeepSeek(modelOptions);
+            }
+            case 'Anthropic': {
+                const { ChatAnthropic } = await import("@langchain/anthropic");
+                return new ChatAnthropic(modelOptions);
+            }
+            default:
+                throw new Error(`Unsupported AI provider: ${aiProvider}`);
+        }
+    } catch (e) {
+        if (e.code === 'ERR_MODULE_NOT_FOUND') {
+            logger.error(`[Assistant] The package for the '${aiProvider}' provider is not installed. Please run 'npm install @langchain/${aiProvider.toLowerCase()}' to use this provider.`);
+            throw new Error(`The AI provider '${aiProvider}' is not installed. Please ask the administrator to install the corresponding package.`);
+        }
+        logger.error(`[Assistant] Error initializing AI provider '${aiProvider}': ${e.message}`);
+        throw e; // Re-throw other errors
+    }
 }

package/src/modules/bucket.js

@@ -266,6 +266,10 @@ export async function onInit(defaultEngine) {
     engine.get('/api/backup/restore', [throttle, middlewareAuthenticator, userInitiator], async (req, res) => {
         const { token, username } = req.query;
 
+        if (!((req.me?.roles || []).includes("admin"))) {
+            return res.status(403).json({success: false, error: 'Cannot dump data.'})
+        }
+
         if (!token || !username) {
             return res.status(400).json({ error: 'Token and username are required.' });
         }

package/src/modules/data/data.cluster.js

@@ -0,0 +1,191 @@
+import { createHash } from 'node:crypto';
+import { clusterPeers, getHost, port } from '../../constants.js';
+import { Config } from '../../config.js';
+import { Logger } from '../../gameObject.js';
+
+const logger = new Logger('DataCluster');
+
+// La liste de tous les nœuds inclut le nœud actuel.
+// On la trie pour s'assurer que l'ordre est le même sur toutes les instances.
+const selfUrl = `http://${getHost()}:${process.env.PORT || port}`;
+const allNodes = Array.from(new Set([selfUrl, ...clusterPeers])).sort();
+const REPLICATION_FACTOR = Config.Get('replicationFactor', 2); // Maître + 1 réplique par défaut
+
+
+logger.info(`[Cluster] Initialized. Self: ${selfUrl}. All nodes: ${allNodes.join(', ')}`);
+
+/**
+ * Diffuse un événement d'invalidation de cache à tous les nœuds du cluster.
+ * @param {string} cacheType - Le type de cache à invalider (ex: 'model').
+ * @param {string} key - La clé spécifique à invalider dans le cache.
+ */
+export async function broadcastCacheInvalidation(cacheType, key) {
+    if (clusterPeers.length === 0) {
+        return; // Pas de cluster, rien à faire.
+    }
+
+    logger.info(`[Cluster] Broadcasting direct cache invalidation for '${cacheType}' with key '${key}' to peers.`);
+
+    // On envoie une requête à chaque autre nœud du cluster.
+    const broadcastPromises = clusterPeers.map(peerUrl => {
+        const targetUrl = `${peerUrl}/api/internal/cache-invalidate`;
+        // On suppose que les nœuds partagent un token/secret pour s'authentifier.
+        // Le token de l'utilisateur actuel est une bonne option s'il est admin ou a des droits étendus.
+        const authToken = (process.env.INTERNAL_CLUSTER_TOKEN) ? `Bearer ${process.env.INTERNAL_CLUSTER_TOKEN}` : null;
+        if (!authToken) {
+            logger.warn(`[Cluster] INTERNAL_CLUSTER_TOKEN is not set. Cannot broadcast cache invalidation securely.`);
+            return Promise.resolve(); // Ne rien faire si pas de token
+        }
+
+        return fetch(targetUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': authToken
+            },
+            body: JSON.stringify({ cacheType, key })
+        }).catch(error => {
+            logger.error(`[Cluster] Failed to send cache invalidation to peer ${peerUrl}: ${error.message}`);
+        });
+    });
+
+    await Promise.allSettled(broadcastPromises);
+}
+
+/**
+ * Détermine la liste ordonnée des nœuds responsables pour un utilisateur.
+ * Le premier est le maître, les suivants sont les répliques.
+ * @param {string} username - Le nom de l'utilisateur.
+ * @returns {string[]} Une liste d'URLs de nœuds.
+ */
+export function getResponsibleNodesForUser(username) {
+    if (allNodes.length <= 1) {
+        return [selfUrl];
+    }
+
+    const numNodes = allNodes.length;
+    const numReplicas = Math.min(REPLICATION_FACTOR, numNodes);
+    const responsibleNodes = [];
+
+    const hash = createHash('sha256').update(username).digest('hex');
+    const startIndex = parseInt(hash.substring(0, 8), 16) % numNodes;
+
+    for (let i = 0; i < numReplicas; i++) {
+        responsibleNodes.push(allNodes[(startIndex + i) % numNodes]);
+    }
+
+    return responsibleNodes;
+}
+
+/**
+ * Détermine l'URL du nœud "maître" pour un utilisateur donné.
+ * @param {string} username - Le nom de l'utilisateur.
+ * @returns {string} L'URL du nœud maître.
+ */
+export function getMasterNodeForUser(username) {
+    return getResponsibleNodesForUser(username)[0];
+}
+
+/**
+ * Vérifie si le nœud actuel est le maître pour cet utilisateur.
+ * @param {string} username - Le nom de l'utilisateur.
+ * @returns {boolean}
+ */
+export function isSelfMasterForUser(username) {
+    return getMasterNodeForUser(username) === selfUrl;
+}
+
+/**
+ * Relaye une requête (lecture ou écriture) vers le nœud maître approprié de manière performante (streaming).
+ * @param {object} req - L'objet requête Express.
+ * @param {object} res - L'objet réponse Express.
+ * @param {string} username - Le nom de l'utilisateur concerné.
+ * @returns {Promise<boolean>} - Retourne `true` si la requête a été relayée, `false` sinon.
+ */
+export async function proxyRequest(req, res, username) {
+    const responsibleNodes = getResponsibleNodesForUser(username);
+    const masterNodeUrl = responsibleNodes[0];
+
+    if (masterNodeUrl === selfUrl) {
+        // Ce nœud est le maître, on ne relaie pas.
+        return false;
+    }
+
+    // Pour les écritures (POST, PUT, PATCH, DELETE), on cible toujours le maître.
+    if (req.method !== 'GET') {
+        logger.info(`[Cluster] Proxying WRITE ${req.method} for user ${username} to master node ${masterNodeUrl}`);
+        await attemptProxy(req, res, masterNodeUrl);
+        return true; // La requête a été traitée (relayée ou a échoué).
+    }
+
+    // Pour les lectures (GET), on tente le maître, puis les répliques en cas d'échec.
+    logger.info(`[Cluster] Proxying READ for user ${username}. Node preference: ${responsibleNodes.join(' -> ')}`);
+    for (const nodeUrl of responsibleNodes) {
+        try {
+            // Si le nœud est nous-mêmes, on arrête le proxying et on laisse le handler local prendre le relais.
+            if (nodeUrl === selfUrl) {
+                logger.info(`[Cluster] Failover to self. Handling request locally.`);
+                return false;
+            }
+            await attemptProxy(req, res, nodeUrl);
+            return true; // Succès, on arrête la boucle de failover.
+        } catch (error) {
+            logger.warn(`[Cluster] Proxy attempt to ${nodeUrl} failed: ${error.message}. Trying next node...`);
+        }
+    }
+
+    // Si tous les nœuds ont échoué
+    logger.error(`[Cluster] All responsible nodes for user ${username} are down. Cannot serve request.`);
+    res.status(503).json({ success: false, error: `Service Unavailable: All responsible nodes for user ${username} are currently down.` });
+    return true; // La requête a été traitée (a échoué).
+}
+
+async function attemptProxy(req, res, nodeUrl) {
+    const targetUrl = new URL(req.originalUrl, nodeUrl);
+    const headers = { ...req.headers };
+    delete headers['host'];
+    headers['X-Federation-Proxy'] = 'true';
+    if (!headers['content-type']) {
+        headers['content-type'] = 'application/json';
+    }
+
+    // Utilisation d'un AbortController pour gérer les timeouts de connexion
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 5000); // Timeout de 5 secondes
+    
+    try {
+        const proxyResponse = await fetch(targetUrl.toString(), {
+            method: req.method,
+            headers: headers,
+            body: (req.method !== 'GET' && req.method !== 'HEAD') ? JSON.stringify(req.fields) : undefined,
+            signal: controller.signal
+        });
+
+        if (!proxyResponse.ok) {
+            throw new Error(`Node responded with status ${proxyResponse.status}`);
+        }
+
+        res.status(proxyResponse.status);
+        proxyResponse.headers.forEach((value, name) => {
+            // On évite de propager des en-têtes liés à la connexion interne du cluster.
+            if (name.toLowerCase() !== 'transfer-encoding' && name.toLowerCase() !== 'connection') {
+                res.setHeader(name, value);
+            }
+        });
+        const { Readable } = await import('node:stream');
+        Readable.fromWeb(proxyResponse.body).pipe(res);
+    } catch (error) {
+        throw error; // Relancer l'erreur pour que la boucle de failover puisse l'attraper.
+    } finally {
+        clearTimeout(timeoutId);
+    }
+}
+
+/**
+ * Vérifie si une requête a déjà été relayée pour éviter les boucles.
+ * @param {object} req - L'objet requête Express.
+ * @returns {boolean}
+ */
+export function isProxiedRequest(req) {
+    return req.headers['x-federation-proxy'] === 'true';
+}

package/src/modules/data/data.core.js

@@ -1,12 +1,9 @@
 import NodeCache from "node-cache";
-import path from "node:path";
 import {Worker} from 'worker_threads';
-import {fileURLToPath} from "node:url";
+import { fileURLToPath, URL } from "node:url";
+import path from "node:path";
 export const modelsCache = new NodeCache( { stdTTL: 100, checkperiod: 120 } );
 
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
 /**
  * Helper to escape characters for regex.
  * @param {string} str The string to escape.
@@ -56,8 +53,12 @@ export let importJobs = {};
  */
 export function runImportExportWorker(action, payload) {
     return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(process.cwd(), './src/workers/import-export-worker.js');
-        const worker = new Worker(workerPath);
+        // Construct a full URL to the worker script. This is the most robust way
+        // to ensure the worker is found, then convert it to a file path.
+        //const workerPath = path.resolve('src/workers/import-export-worker.js');
+        const workerPath = import.meta.dirname;
+
+        const worker = new Worker(path.resolve(workerPath,"../../workers/import-export-worker.js"));
 
         worker.postMessage({ action, payload });
 
@@ -91,7 +92,9 @@ export function runImportExportWorker(action, payload) {
  */
 export function runCryptoWorkerTask(action, payload) {
     return new Promise((resolve, reject) => {
-        const workerPath = path.resolve(__dirname, '../../workers/crypto-worker.js');
+        // Construct a full URL to the worker script, then convert it to a file path
+        // for maximum reliability with the Worker constructor.
+        const workerPath = path.resolve('src/workers/crypto-worker.js');
         const worker = new Worker(workerPath);
 
         worker.postMessage({ action, payload });