@intranefr/superbackend 1.5.1 → 1.5.3

package/src/middleware.js

@@ -1,10 +1,20 @@
 const consoleOverride = require("./services/consoleOverride.service");
-const consoleManager = require("./services/consoleManager.service");
+const { consoleManager } = require("./services/consoleManager.service");
 
 // Initialize console override service early to capture all logs
 // Avoid keeping timers/streams alive during Jest runs.
 if (process.env.NODE_ENV !== "test" && !process.env.JEST_WORKER_ID) {
   consoleOverride.init()
+  
+  // Initialize console manager after a short delay to ensure consoleOverride is fully set up
+  setTimeout(() => {
+    // Set module prefix for this middleware
+    consoleManager.setModulePrefix('middleware');
+    
+    // Initialize console manager early to enable prefixing for all subsequent logs
+    consoleManager.init();
+    console.log("[Console Manager] Initialized - prefixing enabled");
+  }, 20);
 }
 
 const express = require("express");
@@ -119,6 +129,11 @@ function createMiddleware(options = {}) {
       attachTerminalWebsocketServer,
     } = require("./services/terminalsWs.service");
     attachTerminalWebsocketServer(server, { basePathPrefix: adminPath });
+
+    const {
+      attachExperimentsWebsocketServer,
+    } = require("./services/experimentsWs.service");
+    attachExperimentsWebsocketServer(server);
   };
 
   if (!errorCaptureInitialized) {
@@ -146,6 +161,8 @@ function createMiddleware(options = {}) {
       maxPoolSize: 10,
     };
 
+const telegramService = require("./services/telegram.service");
+
     // Return a promise that resolves when connection is established
     const connectionPromise = mongoose
       .connect(mongoUri, connectionOptions)
@@ -156,17 +173,13 @@ function createMiddleware(options = {}) {
         await healthChecksScheduler.start();
         await healthChecksBootstrap.bootstrap();
         await blogCronsBootstrap.bootstrap();
+        await require("./services/experimentsCronsBootstrap.service").bootstrap();
         
-        // Initialize console manager AFTER database is connected
-        if (process.env.NODE_ENV !== "test" && !process.env.JEST_WORKER_ID) {
-          const consoleManagerEnabled = await isConsoleManagerEnabled();
-          if (consoleManagerEnabled) {
-            consoleManager.init();
-            console.log("[Console Manager] Initialized");
-          } else {
-            console.log("[Console Manager] Disabled - console methods not overridden");
-          }
-        }
+        // Initialize Telegram bots
+        await telegramService.init();
+
+        // Console manager is already initialized early in the middleware
+        console.log("[Console Manager] MongoDB connection established");
         
         return true;
       })
@@ -254,12 +267,20 @@ function createMiddleware(options = {}) {
     blogCronsBootstrap.bootstrap().catch((err) => {
       console.error("Failed to bootstrap blog crons:", err);
     });
+
+    require("./services/experimentsCronsBootstrap.service")
+      .bootstrap()
+      .catch((err) => {
+        console.error("Failed to bootstrap experiments crons:", err);
+      });
     
     // Initialize console manager AFTER database is already connected
     if (process.env.NODE_ENV !== "test" && !process.env.JEST_WORKER_ID) {
       isConsoleManagerEnabled().then(consoleManagerEnabled => {
         if (consoleManagerEnabled) {
           consoleManager.init();
+          // Set module prefix after initialization
+          consoleManager.setModulePrefix('middleware');
           console.log("[Console Manager] Initialized");
         } else {
           console.log("[Console Manager] Disabled - console methods not overridden");
@@ -268,6 +289,8 @@ function createMiddleware(options = {}) {
         console.error("[Console Manager] Error checking enabled status:", error);
         console.log("[Console Manager] Fallback to enabled due to error");
         consoleManager.init();
+        // Set module prefix after initialization
+        consoleManager.setModulePrefix('middleware');
         console.log("[Console Manager] Initialized (fallback)");
       });
     }
@@ -467,8 +490,13 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/rbac`, basicAuth, (req, res) => {
-    const templatePath = path.join(__dirname, "..", "views", "admin-rbac.ejs");
+  router.get(`${adminPath}/experiments`, basicAuth, (req, res) => {
+    const templatePath = path.join(
+      __dirname,
+      "..",
+      "views",
+      "admin-experiments.ejs",
+    );
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
         console.error("Error reading template:", err);
@@ -493,13 +521,8 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/terminals`, basicAuth, (req, res) => {
-    const templatePath = path.join(
-      __dirname,
-      "..",
-      "views",
-      "admin-terminals.ejs",
-    );
+  router.get(`${adminPath}/rbac`, basicAuth, (req, res) => {
+    const templatePath = path.join(__dirname, "..", "views", "admin-rbac.ejs");
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
         console.error("Error reading template:", err);
@@ -511,7 +534,6 @@ function createMiddleware(options = {}) {
           {
             baseUrl: req.baseUrl,
             adminPath,
-            endpointRegistry,
           },
           {
             filename: templatePath,
@@ -525,12 +547,12 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/scripts`, basicAuth, (req, res) => {
+  router.get(`${adminPath}/terminals`, basicAuth, (req, res) => {
     const templatePath = path.join(
       __dirname,
       "..",
       "views",
-      "admin-scripts.ejs",
+      "admin-terminals.ejs",
     );
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
@@ -557,8 +579,13 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/crons`, basicAuth, (req, res) => {
-    const templatePath = path.join(__dirname, "..", "views", "admin-crons.ejs");
+  router.get(`${adminPath}/scripts`, basicAuth, (req, res) => {
+    const templatePath = path.join(
+      __dirname,
+      "..",
+      "views",
+      "admin-scripts.ejs",
+    );
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
         console.error("Error reading template:", err);
@@ -570,6 +597,7 @@ function createMiddleware(options = {}) {
           {
             baseUrl: req.baseUrl,
             adminPath,
+            endpointRegistry,
           },
           {
             filename: templatePath,
@@ -583,8 +611,8 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/cache`, basicAuth, (req, res) => {
-    const templatePath = path.join(__dirname, "..", "views", "admin-cache.ejs");
+  router.get(`${adminPath}/crons`, basicAuth, (req, res) => {
+    const templatePath = path.join(__dirname, "..", "views", "admin-crons.ejs");
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
         console.error("Error reading template:", err);
@@ -609,13 +637,8 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/db-browser`, basicAuth, (req, res) => {
-    const templatePath = path.join(
-      __dirname,
-      "..",
-      "views",
-      "admin-db-browser.ejs",
-    );
+  router.get(`${adminPath}/cache`, basicAuth, (req, res) => {
+    const templatePath = path.join(__dirname, "..", "views", "admin-cache.ejs");
     fs.readFile(templatePath, "utf8", (err, template) => {
       if (err) {
         console.error("Error reading template:", err);
@@ -640,6 +663,99 @@ function createMiddleware(options = {}) {
     });
   });
 
+    router.get(`${adminPath}/db-browser`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-db-browser.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
+    });
+
+    router.get(`${adminPath}/telegram`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-telegram.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
+    });
+
+    router.get(`${adminPath}/agents`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-agents.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
+    });
+
   router.use("/api/admin", require("./routes/admin.routes"));
   router.use("/api/admin/settings", require("./routes/globalSettings.routes"));
   router.use(
@@ -650,6 +766,10 @@ function createMiddleware(options = {}) {
     "/api/admin/json-configs",
     require("./routes/adminJsonConfigs.routes"),
   );
+  router.use(
+    "/api/admin/markdowns",
+    require("./routes/adminMarkdowns.routes"),
+  );
   router.use(
     "/api/admin/rate-limits",
     require("./routes/adminRateLimits.routes"),
@@ -677,6 +797,7 @@ function createMiddleware(options = {}) {
     require("./routes/adminDbBrowser.routes"),
   );
   router.use("/api/admin/terminals", require("./routes/adminTerminals.routes"));
+  router.use("/api/admin/experiments", require("./routes/adminExperiments.routes"));
   router.use("/api/admin/assets", require("./routes/adminAssets.routes"));
   router.use(
     "/api/admin/upload-namespaces",
@@ -698,6 +819,8 @@ function createMiddleware(options = {}) {
     require("./routes/adminAudit.routes"),
   );
   router.use("/api/admin/llm", require("./routes/adminLlm.routes"));
+  router.use("/api/admin/telegram", require("./routes/adminTelegram.routes"));
+  router.use("/api/admin/agents", require("./routes/adminAgents.routes"));
   router.use(
     "/api/admin/ejs-virtual",
     require("./routes/adminEjsVirtual.routes"),
@@ -712,6 +835,7 @@ function createMiddleware(options = {}) {
   router.use("/api/settings", require("./routes/globalSettings.routes"));
   router.use("/api/feature-flags", require("./routes/featureFlags.routes"));
   router.use("/api/json-configs", require("./routes/jsonConfigs.routes"));
+  router.use("/api/markdowns", require("./routes/markdowns.routes"));
   router.use("/api/assets", require("./routes/assets.routes"));
   router.use("/api/i18n", require("./routes/i18n.routes"));
   router.use("/api/headless", require("./routes/headless.routes"));
@@ -724,6 +848,7 @@ function createMiddleware(options = {}) {
   router.use("/api/ui-components", require("./routes/uiComponentsPublic.routes"));
   router.use("/api/rbac", require("./routes/rbac.routes"));
   router.use("/api/file-manager", require("./routes/fileManager.routes"));
+  router.use("/api/experiments", require("./routes/experiments.routes"));
 
   // Public blog APIs (headless)
   router.use("/api", require("./routes/blogPublic.routes"));
@@ -731,6 +856,9 @@ function createMiddleware(options = {}) {
   // Internal blog endpoints (used by HTTP CronJobs)
   router.use("/api/internal", require("./routes/blogInternal.routes"));
 
+  // Internal experiments endpoints (used by HTTP CronJobs)
+  router.use("/api/internal", require("./routes/internalExperiments.routes"));
+
   // Public health checks status (gated by global setting)
   router.use(
     "/api/health-checks",
@@ -1311,6 +1439,39 @@ function createMiddleware(options = {}) {
     });
   });
 
+  // Admin markdowns page (protected by basic auth)
+  router.get(`${adminPath}/markdowns`, basicAuth, (req, res) => {
+    const templatePath = path.join(
+      __dirname,
+      "..",
+      "views",
+      "admin-markdowns.ejs",
+    );
+    fs.readFile(templatePath, "utf8", (err, template) => {
+      if (err) {
+        console.error("Error reading template:", err);
+        return res.status(500).send("Error loading page");
+      }
+      try {
+        const html = ejs.render(
+          template,
+          {
+            baseUrl: req.baseUrl,
+            adminPath,
+            endpointRegistry,
+          },
+          {
+            filename: templatePath,
+          },
+        );
+        res.send(html);
+      } catch (renderErr) {
+        console.error("Error rendering template:", renderErr);
+        res.status(500).send("Error rendering page");
+      }
+    });
+  });
+
   // Admin assets page (protected by basic auth)
   router.get(`${adminPath}/assets`, basicAuth, (req, res) => {
     const templatePath = path.join(

package/src/models/Agent.js

@@ -0,0 +1,105 @@
+const mongoose = require('mongoose');
+
+const agentSchema = new mongoose.Schema({
+  name: {
+    type: String,
+    required: true,
+    trim: true
+  },
+  systemPrompt: {
+    type: String,
+    default: `You are a helpful assistant with access to specific tools for querying data.
+
+AVAILABLE TOOLS:
+
+1. query_database: Query the MongoDB database for insights.
+   - Parameters:
+     - modelName (required): The name of the Mongoose model (e.g., User, Markdown, AuditEvent)
+     - query (required): The MongoDB query object
+     - limit (optional): Limit the number of results (default: 5)
+   - Usage: Use this when you need to fetch specific data from the database.
+
+2. get_system_stats: Get general statistics about the system.
+   - Parameters: None
+   - Usage: Use this when you need overall counts of users, markdowns, and other system entities.
+
+3. raw_db_query: Execute raw MongoDB queries for database exploration.
+   - Parameters:
+     - queryType (required): The type of raw query to execute
+       • listDatabases: List all databases (requires admin access)
+       • listCollections: List all collections in a database
+       • countDocuments: Count documents in a collection
+       • findOne: Find a single document in a collection
+       • aggregate: Run aggregation pipeline
+       • adminCommand: Execute admin commands
+     - database (optional): Database name (defaults to current database)
+     - collection (required for collection queries): Collection name
+     - filter (optional): MongoDB filter/query object. Can be:
+       • A JSON object: { createdAt: { $gte: new Date() } }
+       • A JSON string: '{"createdAt": {"$gte": {"$date": "2024-01-01"}}}'
+       • For aggregate: an array of pipeline stages as object or JSON string
+     - limit (optional): Limit results (default: 10)
+     - adminCommand (optional): Admin command for adminCommand queryType (as object or JSON string)
+   - Usage: Use this to discover collection names, databases, or run admin commands.
+   - IMPORTANT: For complex queries, use JSON string format to avoid parsing issues
+
+IMPORTANT ERROR HANDLING INSTRUCTIONS:
+- When a tool returns an error, it will be in structured JSON format with error details
+- ALWAYS provide a friendly, conversational response to the user about tool errors
+- NEVER show raw error JSON to users
+- DO: "I had trouble accessing the database. Let me try a different approach..."
+- DO NOT: Show the actual error JSON to users
+- Extract the error message and provide helpful suggestions based on the error context
+- If an error is not recoverable, explain why and suggest alternatives
+- If an error is recoverable, explain what you'll try next
+- Use the error suggestions provided in the tool response to inform your response
+
+INSTRUCTIONS:
+- Always use tools when you need actual data from the database
+- Never make up data or statistics
+- For database queries, use exact model names as they appear in the system
+- When using query_database, construct appropriate MongoDB query objects based on the user's request
+- If you don't have enough information for a query, ask clarifying questions
+- Use get_system_stats for high-level overview requests
+- Use raw_db_query for:
+  * Discovering what collections exist: queryType: "listCollections"
+  * Finding database names: queryType: "listDatabases" (may require admin)
+  * Counting documents: queryType: "countDocuments" with collection and filter
+  * Exploring collection structure: queryType: "findOne" or "aggregate"
+- For specific records, use query_database with appropriate filters
+
+Respond helpfully and only use the tools when necessary for accurate information. Always provide friendly error messages to users when tools fail.`
+  },
+  providerKey: {
+    type: String,
+    required: true
+  },
+  model: {
+    type: String,
+    required: true
+  },
+  tools: {
+    type: [String],
+    default: []
+  },
+  temperature: {
+    type: Number,
+    default: 0.7
+  },
+  maxIterations: {
+    type: Number,
+    default: 10
+  },
+  orgId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'Organization'
+  },
+  ownerUserId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'User'
+  }
+}, {
+  timestamps: true
+});
+
+module.exports = mongoose.model('Agent', agentSchema);

package/src/models/AgentMessage.js

@@ -0,0 +1,82 @@
+const mongoose = require('mongoose');
+const { ObjectId } = mongoose.Schema.Types;
+
+const agentMessageSchema = new mongoose.Schema({
+  agentId: {
+    type: ObjectId,
+    ref: 'Agent',
+    required: true,
+    index: true
+  },
+  chatId: {
+    type: String,
+    required: true,
+    index: true
+  },
+  role: {
+    type: String,
+    enum: ['user', 'assistant', 'system', 'tool'],
+    required: true
+  },
+  content: {
+    type: String,
+    required: function() {
+      // Content is not required if:
+      // 1. Role is 'tool' (content might be in metadata or implied) - though usually tool has content
+      // 2. Role is 'assistant' AND it has toolCalls (OpenAI often returns null content with tool calls)
+      if (this.role === 'tool') return false;
+      if (this.role === 'assistant' && this.toolCalls && this.toolCalls.length > 0) return false;
+      return true;
+    }
+  },
+  toolCalls: [{
+    name: String,
+    arguments: mongoose.Schema.Types.Mixed,
+    toolCallId: String
+  }],
+  toolCallId: {
+    type: String,
+    index: true
+  },
+  metadata: {
+    tokens: Number,
+    processingTime: Number,
+    model: String,
+    provider: String,
+    timestamp: Date,
+    temperature: Number
+  },
+  createdAt: {
+    type: Date,
+    default: Date.now,
+    index: true
+  },
+  updatedAt: {
+    type: Date,
+    default: Date.now
+  }
+}, {
+  timestamps: true,
+  toJSON: { virtuals: true },
+  toObject: { virtuals: true }
+});
+
+// Compound index for efficient session history retrieval
+agentMessageSchema.index(
+  { agentId: 1, chatId: 1, createdAt: 1 },
+  { name: 'session_history_idx' }
+);
+
+// Index for tool call lookup
+agentMessageSchema.index(
+  { toolCallId: 1 },
+  { name: 'tool_call_idx' }
+);
+
+// Index for searching content
+agentMessageSchema.index(
+  { content: 'text' },
+  { name: 'content_search_idx' }
+);
+
+module.exports = mongoose.model('AgentMessage', agentMessageSchema);

package/src/models/CacheEntry.js

@@ -10,7 +10,7 @@ const cacheEntrySchema = new mongoose.Schema(
 
     sizeBytes: { type: Number, default: 0 },
 
-    expiresAt: { type: Date, default: null, index: true },
+    expiresAt: { type: Date, default: null },
 
     hits: { type: Number, default: 0 },
     lastAccessAt: { type: Date, default: null },

package/src/models/ConsoleLog.js

@@ -12,7 +12,7 @@ const consoleLogSchema = new mongoose.Schema(
     requestId: { type: String, default: '', index: true },
 
     createdAt: { type: Date, default: Date.now, index: true },
-    expiresAt: { type: Date, default: null, index: true },
+    expiresAt: { type: Date, default: null },
   },
   { timestamps: false, collection: 'console_logs' },
 );

package/src/models/Experiment.js

@@ -0,0 +1,75 @@
+const mongoose = require('mongoose');
+
+const experimentVariantSchema = new mongoose.Schema(
+  {
+    key: { type: String, required: true },
+    weight: { type: Number, default: 0 },
+    configSlug: { type: String, default: '' },
+  },
+  { _id: false },
+);
+
+const metricDefinitionSchema = new mongoose.Schema(
+  {
+    key: { type: String, required: true },
+    kind: { type: String, enum: ['count', 'sum', 'avg', 'rate'], default: 'count' },
+    numeratorEventKey: { type: String, default: '' },
+    denominatorEventKey: { type: String, default: '' },
+    objective: { type: String, enum: ['maximize', 'minimize'], default: 'maximize' },
+  },
+  { _id: false },
+);
+
+const winnerPolicySchema = new mongoose.Schema(
+  {
+    mode: { type: String, enum: ['manual', 'automatic'], default: 'manual' },
+    pickAfterMs: { type: Number, default: 0 },
+    minAssignments: { type: Number, default: 0 },
+    minExposures: { type: Number, default: 0 },
+    minConversions: { type: Number, default: 0 },
+    statMethod: { type: String, enum: ['simple_rate', 'bayesian_beta'], default: 'simple_rate' },
+    overrideWinnerVariantKey: { type: String, default: '' },
+  },
+  { _id: false },
+);
+
+const experimentSchema = new mongoose.Schema(
+  {
+    organizationId: { type: mongoose.Schema.Types.ObjectId, ref: 'Organization', default: null, index: true },
+
+    code: { type: String, required: true },
+    name: { type: String, default: '' },
+    description: { type: String, default: '' },
+
+    status: { type: String, enum: ['draft', 'running', 'paused', 'completed'], default: 'draft', index: true },
+
+    startedAt: { type: Date, default: null },
+    endsAt: { type: Date, default: null },
+
+    assignment: {
+      unit: { type: String, enum: ['subjectId'], default: 'subjectId' },
+      sticky: { type: Boolean, default: true },
+      salt: { type: String, default: '' },
+    },
+
+    variants: { type: [experimentVariantSchema], default: [] },
+
+    primaryMetric: { type: metricDefinitionSchema, required: true },
+    secondaryMetrics: { type: [metricDefinitionSchema], default: [] },
+
+    winnerPolicy: { type: winnerPolicySchema, default: () => ({}) },
+
+    winnerVariantKey: { type: String, default: '' },
+    winnerDecidedAt: { type: Date, default: null },
+    winnerReason: { type: String, default: '' },
+
+    createdByUserId: { type: mongoose.Schema.Types.ObjectId, ref: 'User', default: null },
+    updatedByUserId: { type: mongoose.Schema.Types.ObjectId, ref: 'User', default: null },
+  },
+  { timestamps: true, collection: 'experiments' },
+);
+
+experimentSchema.index({ organizationId: 1, code: 1 }, { unique: true });
+experimentSchema.index({ status: 1, startedAt: 1 });
+
+module.exports = mongoose.models.Experiment || mongoose.model('Experiment', experimentSchema);