@intranefr/superbackend 1.5.2 → 1.5.3

package/src/middleware.js

@@ -161,6 +161,8 @@ function createMiddleware(options = {}) {
       maxPoolSize: 10,
     };
 
+const telegramService = require("./services/telegram.service");
+
     // Return a promise that resolves when connection is established
     const connectionPromise = mongoose
       .connect(mongoUri, connectionOptions)
@@ -173,6 +175,9 @@ function createMiddleware(options = {}) {
         await blogCronsBootstrap.bootstrap();
         await require("./services/experimentsCronsBootstrap.service").bootstrap();
         
+        // Initialize Telegram bots
+        await telegramService.init();
+
         // Console manager is already initialized early in the middleware
         console.log("[Console Manager] MongoDB connection established");
         
@@ -658,36 +663,98 @@ function createMiddleware(options = {}) {
     });
   });
 
-  router.get(`${adminPath}/db-browser`, basicAuth, (req, res) => {
-    const templatePath = path.join(
-      __dirname,
-      "..",
-      "views",
-      "admin-db-browser.ejs",
-    );
-    fs.readFile(templatePath, "utf8", (err, template) => {
-      if (err) {
-        console.error("Error reading template:", err);
-        return res.status(500).send("Error loading page");
-      }
-      try {
-        const html = ejs.render(
-          template,
-          {
-            baseUrl: req.baseUrl,
-            adminPath,
-          },
-          {
-            filename: templatePath,
-          },
-        );
-        res.send(html);
-      } catch (renderErr) {
-        console.error("Error rendering template:", renderErr);
-        res.status(500).send("Error rendering page");
-      }
+    router.get(`${adminPath}/db-browser`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-db-browser.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
+    });
+
+    router.get(`${adminPath}/telegram`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-telegram.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
+    });
+
+    router.get(`${adminPath}/agents`, basicAuth, (req, res) => {
+      const templatePath = path.join(
+        __dirname,
+        "..",
+        "views",
+        "admin-agents.ejs",
+      );
+      fs.readFile(templatePath, "utf8", (err, template) => {
+        if (err) {
+          console.error("Error reading template:", err);
+          return res.status(500).send("Error loading page");
+        }
+        try {
+          const html = ejs.render(
+            template,
+            {
+              baseUrl: req.baseUrl,
+              adminPath,
+            },
+            {
+              filename: templatePath,
+            },
+          );
+          res.send(html);
+        } catch (renderErr) {
+          console.error("Error rendering template:", renderErr);
+          res.status(500).send("Error rendering page");
+        }
+      });
     });
-  });
 
   router.use("/api/admin", require("./routes/admin.routes"));
   router.use("/api/admin/settings", require("./routes/globalSettings.routes"));
@@ -699,6 +766,10 @@ function createMiddleware(options = {}) {
     "/api/admin/json-configs",
     require("./routes/adminJsonConfigs.routes"),
   );
+  router.use(
+    "/api/admin/markdowns",
+    require("./routes/adminMarkdowns.routes"),
+  );
   router.use(
     "/api/admin/rate-limits",
     require("./routes/adminRateLimits.routes"),
@@ -748,6 +819,8 @@ function createMiddleware(options = {}) {
     require("./routes/adminAudit.routes"),
   );
   router.use("/api/admin/llm", require("./routes/adminLlm.routes"));
+  router.use("/api/admin/telegram", require("./routes/adminTelegram.routes"));
+  router.use("/api/admin/agents", require("./routes/adminAgents.routes"));
   router.use(
     "/api/admin/ejs-virtual",
     require("./routes/adminEjsVirtual.routes"),
@@ -762,6 +835,7 @@ function createMiddleware(options = {}) {
   router.use("/api/settings", require("./routes/globalSettings.routes"));
   router.use("/api/feature-flags", require("./routes/featureFlags.routes"));
   router.use("/api/json-configs", require("./routes/jsonConfigs.routes"));
+  router.use("/api/markdowns", require("./routes/markdowns.routes"));
   router.use("/api/assets", require("./routes/assets.routes"));
   router.use("/api/i18n", require("./routes/i18n.routes"));
   router.use("/api/headless", require("./routes/headless.routes"));
@@ -1365,6 +1439,39 @@ function createMiddleware(options = {}) {
     });
   });
 
+  // Admin markdowns page (protected by basic auth)
+  router.get(`${adminPath}/markdowns`, basicAuth, (req, res) => {
+    const templatePath = path.join(
+      __dirname,
+      "..",
+      "views",
+      "admin-markdowns.ejs",
+    );
+    fs.readFile(templatePath, "utf8", (err, template) => {
+      if (err) {
+        console.error("Error reading template:", err);
+        return res.status(500).send("Error loading page");
+      }
+      try {
+        const html = ejs.render(
+          template,
+          {
+            baseUrl: req.baseUrl,
+            adminPath,
+            endpointRegistry,
+          },
+          {
+            filename: templatePath,
+          },
+        );
+        res.send(html);
+      } catch (renderErr) {
+        console.error("Error rendering template:", renderErr);
+        res.status(500).send("Error rendering page");
+      }
+    });
+  });
+
   // Admin assets page (protected by basic auth)
   router.get(`${adminPath}/assets`, basicAuth, (req, res) => {
     const templatePath = path.join(

package/src/models/Agent.js

@@ -0,0 +1,105 @@
+const mongoose = require('mongoose');
+
+const agentSchema = new mongoose.Schema({
+  name: {
+    type: String,
+    required: true,
+    trim: true
+  },
+  systemPrompt: {
+    type: String,
+    default: `You are a helpful assistant with access to specific tools for querying data.
+
+AVAILABLE TOOLS:
+
+1. query_database: Query the MongoDB database for insights.
+   - Parameters:
+     - modelName (required): The name of the Mongoose model (e.g., User, Markdown, AuditEvent)
+     - query (required): The MongoDB query object
+     - limit (optional): Limit the number of results (default: 5)
+   - Usage: Use this when you need to fetch specific data from the database.
+
+2. get_system_stats: Get general statistics about the system.
+   - Parameters: None
+   - Usage: Use this when you need overall counts of users, markdowns, and other system entities.
+
+3. raw_db_query: Execute raw MongoDB queries for database exploration.
+   - Parameters:
+     - queryType (required): The type of raw query to execute
+       • listDatabases: List all databases (requires admin access)
+       • listCollections: List all collections in a database
+       • countDocuments: Count documents in a collection
+       • findOne: Find a single document in a collection
+       • aggregate: Run aggregation pipeline
+       • adminCommand: Execute admin commands
+     - database (optional): Database name (defaults to current database)
+     - collection (required for collection queries): Collection name
+     - filter (optional): MongoDB filter/query object. Can be:
+       • A JSON object: { createdAt: { $gte: new Date() } }
+       • A JSON string: '{"createdAt": {"$gte": {"$date": "2024-01-01"}}}'
+       • For aggregate: an array of pipeline stages as object or JSON string
+     - limit (optional): Limit results (default: 10)
+     - adminCommand (optional): Admin command for adminCommand queryType (as object or JSON string)
+   - Usage: Use this to discover collection names, databases, or run admin commands.
+   - IMPORTANT: For complex queries, use JSON string format to avoid parsing issues
+
+IMPORTANT ERROR HANDLING INSTRUCTIONS:
+- When a tool returns an error, it will be in structured JSON format with error details
+- ALWAYS provide a friendly, conversational response to the user about tool errors
+- NEVER show raw error JSON to users
+- DO: "I had trouble accessing the database. Let me try a different approach..."
+- DO NOT: Show the actual error JSON to users
+- Extract the error message and provide helpful suggestions based on the error context
+- If an error is not recoverable, explain why and suggest alternatives
+- If an error is recoverable, explain what you'll try next
+- Use the error suggestions provided in the tool response to inform your response
+
+INSTRUCTIONS:
+- Always use tools when you need actual data from the database
+- Never make up data or statistics
+- For database queries, use exact model names as they appear in the system
+- When using query_database, construct appropriate MongoDB query objects based on the user's request
+- If you don't have enough information for a query, ask clarifying questions
+- Use get_system_stats for high-level overview requests
+- Use raw_db_query for:
+  * Discovering what collections exist: queryType: "listCollections"
+  * Finding database names: queryType: "listDatabases" (may require admin)
+  * Counting documents: queryType: "countDocuments" with collection and filter
+  * Exploring collection structure: queryType: "findOne" or "aggregate"
+- For specific records, use query_database with appropriate filters
+
+Respond helpfully and only use the tools when necessary for accurate information. Always provide friendly error messages to users when tools fail.`
+  },
+  providerKey: {
+    type: String,
+    required: true
+  },
+  model: {
+    type: String,
+    required: true
+  },
+  tools: {
+    type: [String],
+    default: []
+  },
+  temperature: {
+    type: Number,
+    default: 0.7
+  },
+  maxIterations: {
+    type: Number,
+    default: 10
+  },
+  orgId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'Organization'
+  },
+  ownerUserId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'User'
+  }
+}, {
+  timestamps: true
+});
+
+module.exports = mongoose.model('Agent', agentSchema);

package/src/models/AgentMessage.js

@@ -0,0 +1,82 @@
+const mongoose = require('mongoose');
+const { ObjectId } = mongoose.Schema.Types;
+
+const agentMessageSchema = new mongoose.Schema({
+  agentId: {
+    type: ObjectId,
+    ref: 'Agent',
+    required: true,
+    index: true
+  },
+  chatId: {
+    type: String,
+    required: true,
+    index: true
+  },
+  role: {
+    type: String,
+    enum: ['user', 'assistant', 'system', 'tool'],
+    required: true
+  },
+  content: {
+    type: String,
+    required: function() {
+      // Content is not required if:
+      // 1. Role is 'tool' (content might be in metadata or implied) - though usually tool has content
+      // 2. Role is 'assistant' AND it has toolCalls (OpenAI often returns null content with tool calls)
+      if (this.role === 'tool') return false;
+      if (this.role === 'assistant' && this.toolCalls && this.toolCalls.length > 0) return false;
+      return true;
+    }
+  },
+  toolCalls: [{
+    name: String,
+    arguments: mongoose.Schema.Types.Mixed,
+    toolCallId: String
+  }],
+  toolCallId: {
+    type: String,
+    index: true
+  },
+  metadata: {
+    tokens: Number,
+    processingTime: Number,
+    model: String,
+    provider: String,
+    timestamp: Date,
+    temperature: Number
+  },
+  createdAt: {
+    type: Date,
+    default: Date.now,
+    index: true
+  },
+  updatedAt: {
+    type: Date,
+    default: Date.now
+  }
+}, {
+  timestamps: true,
+  toJSON: { virtuals: true },
+  toObject: { virtuals: true }
+});
+
+// Compound index for efficient session history retrieval
+agentMessageSchema.index(
+  { agentId: 1, chatId: 1, createdAt: 1 },
+  { name: 'session_history_idx' }
+);
+
+// Index for tool call lookup
+agentMessageSchema.index(
+  { toolCallId: 1 },
+  { name: 'tool_call_idx' }
+);
+
+// Index for searching content
+agentMessageSchema.index(
+  { content: 'text' },
+  { name: 'content_search_idx' }
+);
+
+module.exports = mongoose.model('AgentMessage', agentMessageSchema);

package/src/models/Markdown.js

@@ -0,0 +1,75 @@
+const mongoose = require('mongoose');
+
+const markdownSchema = new mongoose.Schema(
+  {
+    title: {
+      type: String,
+      required: true,
+      trim: true,
+    },
+    slug: {
+      type: String,
+      required: true,
+      index: true,
+      trim: true,
+    },
+    category: {
+      type: String,
+      required: true,
+      index: true,
+      trim: true,
+      default: 'general',
+    },
+    group_code: {
+      type: String,
+      required: false,
+      index: true,
+      trim: true,
+      default: '',
+    },
+    markdownRaw: {
+      type: String,
+      required: true,
+      default: '',
+    },
+    publicEnabled: {
+      type: Boolean,
+      default: false,
+      index: true,
+    },
+    cacheTtlSeconds: {
+      type: Number,
+      default: 0,
+    },
+    status: {
+      type: String,
+      enum: ['draft', 'published', 'archived'],
+      default: 'draft',
+      index: true,
+    },
+    ownerUserId: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'User',
+      index: true,
+      default: null,
+    },
+    orgId: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'Organization',
+      index: true,
+      default: null,
+    },
+  },
+  { timestamps: true },
+);
+
+// Compound unique index for fast lookups
+markdownSchema.index({ category: 1, group_code: 1, slug: 1 }, { unique: true });
+
+// Additional indexes for common queries
+markdownSchema.index({ status: 1, publicEnabled: 1 });
+markdownSchema.index({ category: 1, status: 1 });
+markdownSchema.index({ ownerUserId: 1, createdAt: -1 });
+markdownSchema.index({ orgId: 1, createdAt: -1 });
+
+module.exports = mongoose.model('Markdown', markdownSchema);

package/src/models/ScriptRun.js

@@ -14,6 +14,14 @@ const scriptRunSchema = new mongoose.Schema(
     finishedAt: { type: Date, default: null },
     exitCode: { type: Number, default: null },
     outputTail: { type: String, default: '' },
+    fullOutput: { type: String, default: '' },
+    programmaticOutput: { type: String, default: '' },
+    returnResult: { type: String, default: '' },
+    lastConsoleLog: { type: String, default: '' },
+    outputType: { type: String, enum: ['return', 'console', 'none'], default: 'none' },
+    outputSize: { type: Number, default: 0 },
+    lineCount: { type: Number, default: 0 },
+    lastOutputUpdate: { type: Date, default: null },
     meta: { type: mongoose.Schema.Types.Mixed, default: null },
   },
   { timestamps: true, collection: 'script_runs' },

package/src/models/TelegramBot.js

@@ -0,0 +1,42 @@
+const mongoose = require('mongoose');
+
+const telegramBotSchema = new mongoose.Schema({
+  name: {
+    type: String,
+    required: true,
+    trim: true
+  },
+  token: {
+    type: String,
+    required: true,
+    trim: true
+  },
+  isActive: {
+    type: Boolean,
+    default: false
+  },
+  allowedUserIds: {
+    type: [String],
+    default: []
+  },
+  defaultAgentId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'Agent'
+  },
+  orgId: {
+    type: mongoose.Schema.Types.ObjectId,
+    ref: 'Organization'
+  },
+  status: {
+    type: String,
+    enum: ['stopped', 'running', 'error'],
+    default: 'stopped'
+  },
+  lastError: {
+    type: String
+  }
+}, {
+  timestamps: true
+});
+
+module.exports = mongoose.model('TelegramBot', telegramBotSchema);

package/src/routes/adminAgents.routes.js

@@ -0,0 +1,13 @@
+const express = require('express');
+const router = express.Router();
+const adminAgentsController = require('../controllers/adminAgents.controller');
+const { basicAuth } = require('../middleware/auth');
+
+router.use(basicAuth);
+
+router.get('/', adminAgentsController.listAgents);
+router.post('/', adminAgentsController.createAgent);
+router.put('/:id', adminAgentsController.updateAgent);
+router.delete('/:id', adminAgentsController.deleteAgent);
+
+module.exports = router;

package/src/routes/adminLlm.routes.js

@@ -5,6 +5,7 @@ const adminLlmController = require("../controllers/adminLlm.controller");
 const rateLimiter = require("../services/rateLimiter.service");
 
 router.get("/config", basicAuth, adminLlmController.getConfig);
+router.get("/providers", basicAuth, adminLlmController.listProviders);
 router.post("/config", basicAuth, rateLimiter.limit("llmConfigLimiter"), adminLlmController.saveConfig);
 router.get("/openrouter/models", basicAuth, adminLlmController.listOpenRouterModels);
 router.post("/prompts/:key/test", basicAuth, rateLimiter.limit("llmConfigLimiter"), adminLlmController.testPrompt);

package/src/routes/adminMarkdowns.routes.js

@@ -0,0 +1,16 @@
+const express = require('express');
+const router = express.Router();
+const { basicAuth } = require('../middleware/auth');
+
+const adminMarkdownsController = require('../controllers/adminMarkdowns.controller');
+
+router.get('/', basicAuth, adminMarkdownsController.list);
+router.get('/group-codes/:category', basicAuth, adminMarkdownsController.getGroupCodes);
+router.get('/folder/:category/:group_code?', basicAuth, adminMarkdownsController.getFolderContents);
+router.get('/:id', basicAuth, adminMarkdownsController.get);
+router.post('/', basicAuth, adminMarkdownsController.create);
+router.put('/:id', basicAuth, adminMarkdownsController.update);
+router.delete('/:id', basicAuth, adminMarkdownsController.remove);
+router.post('/validate-path', basicAuth, adminMarkdownsController.validatePath);
+
+module.exports = router;

package/src/routes/adminScripts.routes.js

@@ -10,7 +10,10 @@ router.get('/', controller.listScripts);
 router.post('/', controller.createScript);
 router.get('/runs', controller.listRuns);
 router.get('/runs/:runId', controller.getRun);
-router.get('/runs/:runId/stream', controller.streamRun);
+router.get('/runs/:runId/stream', controller.streamRunLogs);
+router.get('/runs/:runId/programmatic-output', controller.getProgrammaticOutput);
+router.get('/runs/:runId/full-output', controller.getFullOutput);
+router.get('/runs/:runId/download', controller.downloadOutput);
 
 router.get('/:id', controller.getScript);
 router.put('/:id', controller.updateScript);

package/src/routes/adminTelegram.routes.js

@@ -0,0 +1,14 @@
+const express = require('express');
+const router = express.Router();
+const adminTelegramController = require('../controllers/adminTelegram.controller');
+const { basicAuth } = require('../middleware/auth');
+
+router.use(basicAuth);
+
+router.get('/', adminTelegramController.listBots);
+router.post('/', adminTelegramController.createBot);
+router.put('/:id', adminTelegramController.updateBot);
+router.delete('/:id', adminTelegramController.deleteBot);
+router.post('/:id/toggle', adminTelegramController.toggleBot);
+
+module.exports = router;

package/src/routes/markdowns.routes.js

@@ -0,0 +1,16 @@
+const express = require('express');
+const router = express.Router();
+
+const markdownsController = require('../controllers/markdowns.controller');
+
+// JSON versions (more specific first)
+router.get('/:category/:group_code/:slug/json', markdownsController.getByPath);
+router.get('/:category/:slug/json', markdownsController.getByPath);
+
+// Raw versions
+router.get('/:category/:group_code/:slug', markdownsController.getByPath);
+router.get('/:category/:slug', markdownsController.getByPath); // No group_code
+
+router.get('/search', markdownsController.search);
+
+module.exports = router;