@intranefr/superbackend 1.5.1 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/.env.example +10 -0
  2. package/index.js +2 -0
  3. package/manage.js +745 -0
  4. package/package.json +5 -2
  5. package/src/controllers/admin.controller.js +79 -6
  6. package/src/controllers/adminAgents.controller.js +37 -0
  7. package/src/controllers/adminExperiments.controller.js +200 -0
  8. package/src/controllers/adminLlm.controller.js +19 -0
  9. package/src/controllers/adminMarkdowns.controller.js +157 -0
  10. package/src/controllers/adminScripts.controller.js +243 -74
  11. package/src/controllers/adminTelegram.controller.js +72 -0
  12. package/src/controllers/experiments.controller.js +85 -0
  13. package/src/controllers/internalExperiments.controller.js +17 -0
  14. package/src/controllers/markdowns.controller.js +42 -0
  15. package/src/helpers/mongooseHelper.js +258 -0
  16. package/src/helpers/scriptBase.js +230 -0
  17. package/src/helpers/scriptRunner.js +335 -0
  18. package/src/middleware.js +195 -34
  19. package/src/models/Agent.js +105 -0
  20. package/src/models/AgentMessage.js +82 -0
  21. package/src/models/CacheEntry.js +1 -1
  22. package/src/models/ConsoleLog.js +1 -1
  23. package/src/models/Experiment.js +75 -0
  24. package/src/models/ExperimentAssignment.js +23 -0
  25. package/src/models/ExperimentEvent.js +26 -0
  26. package/src/models/ExperimentMetricBucket.js +30 -0
  27. package/src/models/GlobalSetting.js +1 -2
  28. package/src/models/Markdown.js +75 -0
  29. package/src/models/RateLimitCounter.js +1 -1
  30. package/src/models/ScriptDefinition.js +1 -0
  31. package/src/models/ScriptRun.js +8 -0
  32. package/src/models/TelegramBot.js +42 -0
  33. package/src/models/Webhook.js +2 -0
  34. package/src/routes/admin.routes.js +2 -0
  35. package/src/routes/adminAgents.routes.js +13 -0
  36. package/src/routes/adminConsoleManager.routes.js +1 -1
  37. package/src/routes/adminExperiments.routes.js +29 -0
  38. package/src/routes/adminLlm.routes.js +1 -0
  39. package/src/routes/adminMarkdowns.routes.js +16 -0
  40. package/src/routes/adminScripts.routes.js +4 -1
  41. package/src/routes/adminTelegram.routes.js +14 -0
  42. package/src/routes/blogInternal.routes.js +2 -2
  43. package/src/routes/experiments.routes.js +30 -0
  44. package/src/routes/internalExperiments.routes.js +15 -0
  45. package/src/routes/markdowns.routes.js +16 -0
  46. package/src/services/agent.service.js +546 -0
  47. package/src/services/agentHistory.service.js +345 -0
  48. package/src/services/agentTools.service.js +578 -0
  49. package/src/services/blogCronsBootstrap.service.js +7 -6
  50. package/src/services/consoleManager.service.js +56 -18
  51. package/src/services/consoleOverride.service.js +1 -0
  52. package/src/services/experiments.service.js +273 -0
  53. package/src/services/experimentsAggregation.service.js +308 -0
  54. package/src/services/experimentsCronsBootstrap.service.js +118 -0
  55. package/src/services/experimentsRetention.service.js +43 -0
  56. package/src/services/experimentsWs.service.js +134 -0
  57. package/src/services/globalSettings.service.js +15 -0
  58. package/src/services/jsonConfigs.service.js +24 -12
  59. package/src/services/llm.service.js +219 -6
  60. package/src/services/markdowns.service.js +522 -0
  61. package/src/services/scriptsRunner.service.js +514 -23
  62. package/src/services/telegram.service.js +130 -0
  63. package/src/utils/rbac/rightsRegistry.js +4 -0
  64. package/views/admin-agents.ejs +273 -0
  65. package/views/admin-coolify-deploy.ejs +8 -8
  66. package/views/admin-dashboard.ejs +63 -12
  67. package/views/admin-experiments.ejs +91 -0
  68. package/views/admin-markdowns.ejs +905 -0
  69. package/views/admin-scripts.ejs +817 -6
  70. package/views/admin-telegram.ejs +269 -0
  71. package/views/partials/dashboard/nav-items.ejs +4 -0
  72. package/views/partials/dashboard/palette.ejs +5 -3
  73. package/src/middleware/internalCronAuth.js +0 -29
@@ -0,0 +1,258 @@
1
+ const mongoose = require('mongoose');
2
+
3
+ /**
4
+ * Mongoose connection state management
5
+ * Provides centralized connection handling with reference counting and automatic cleanup
6
+ */
7
+ class MongooseHelper {
8
+ constructor() {
9
+ this.connectionPromise = null;
10
+ this.isConnected = false;
11
+ this.connectionCount = 0;
12
+ this.connectionOptions = null;
13
+ }
14
+
15
+ /**
16
+ * Get MongoDB URI from environment with fallbacks
17
+ * @returns {string} MongoDB connection URI
18
+ */
19
+ getMongoUri() {
20
+ const uri = process.env.MONGODB_URI || process.env.MONGO_URI || 'mongodb://localhost:27017/myappdb';
21
+ if (!uri) {
22
+ throw new Error('Missing MONGODB_URI or MONGO_URI environment variable');
23
+ }
24
+ return uri;
25
+ }
26
+
27
+ /**
28
+ * Get mongoose connection options
29
+ * @returns {Object} Connection options
30
+ */
31
+ getConnectionOptions() {
32
+ if (this.connectionOptions) {
33
+ return this.connectionOptions;
34
+ }
35
+
36
+ this.connectionOptions = {
37
+ serverSelectionTimeoutMS: 5000,
38
+ maxPoolSize: 2, // Conservative for scripts
39
+ bufferCommands: false,
40
+ // Add retry settings for reliability
41
+ retryWrites: true,
42
+ retryReads: true,
43
+ // Add socket settings for scripts
44
+ socketTimeoutMS: 30000,
45
+ connectTimeoutMS: 10000,
46
+ };
47
+
48
+ return this.connectionOptions;
49
+ }
50
+
51
+ /**
52
+ * Connect to MongoDB (singleton pattern)
53
+ * @returns {Promise<mongoose.Connection>} Mongoose connection
54
+ */
55
+ async connect() {
56
+ // Return existing connection if already connected
57
+ if (this.isConnected && mongoose.connection.readyState === 1) {
58
+ this.connectionCount++;
59
+ return mongoose.connection;
60
+ }
61
+
62
+ // Return existing promise if connection is in progress
63
+ if (this.connectionPromise) {
64
+ await this.connectionPromise;
65
+ this.connectionCount++;
66
+ return mongoose.connection;
67
+ }
68
+
69
+ // Create new connection promise
70
+ this.connectionPromise = this._createConnection();
71
+ await this.connectionPromise;
72
+ this.connectionCount++;
73
+ return mongoose.connection;
74
+ }
75
+
76
+ /**
77
+ * Internal connection creation
78
+ * @private
79
+ * @returns {Promise<mongoose.Connection>}
80
+ */
81
+ async _createConnection() {
82
+ try {
83
+ const uri = this.getMongoUri();
84
+ const options = this.getConnectionOptions();
85
+
86
+ if (!process.env.TUI_MODE) console.log(`[MongooseHelper] Connecting to MongoDB...`);
87
+
88
+ // Clear any existing connection
89
+ if (mongoose.connection.readyState !== 0) {
90
+ await mongoose.disconnect();
91
+ }
92
+
93
+ await mongoose.connect(uri, options);
94
+
95
+ this.isConnected = true;
96
+
97
+ if (!process.env.TUI_MODE) console.log(`[MongooseHelper] ✅ Connected to MongoDB`);
98
+
99
+ // Setup connection error handling
100
+ mongoose.connection.on('error', (error) => {
101
+ console.error('[MongooseHelper] Connection error:', error);
102
+ this.isConnected = false;
103
+ this.connectionPromise = null;
104
+ });
105
+
106
+ mongoose.connection.on('disconnected', () => {
107
+ if (!process.env.TUI_MODE) console.log('[MongooseHelper] Disconnected from MongoDB');
108
+ this.isConnected = false;
109
+ this.connectionPromise = null;
110
+ });
111
+
112
+ mongoose.connection.on('reconnected', () => {
113
+ if (!process.env.TUI_MODE) console.log('[MongooseHelper] Reconnected to MongoDB');
114
+ this.isConnected = true;
115
+ });
116
+
117
+ return mongoose.connection;
118
+ } catch (error) {
119
+ this.connectionPromise = null;
120
+ throw error;
121
+ }
122
+ }
123
+
124
+ /**
125
+ * Disconnect from MongoDB (reference counting)
126
+ * @returns {Promise<void>}
127
+ */
128
+ async disconnect() {
129
+ if (!this.isConnected) {
130
+ return;
131
+ }
132
+
133
+ this.connectionCount--;
134
+
135
+ // Only disconnect if no more references
136
+ if (this.connectionCount <= 0) {
137
+ try {
138
+ await mongoose.disconnect();
139
+ if (!process.env.TUI_MODE) console.log('[MongooseHelper] ✅ Disconnected from MongoDB');
140
+ } catch (error) {
141
+ console.error('[MongooseHelper] Disconnect error:', error);
142
+ } finally {
143
+ this.isConnected = false;
144
+ this.connectionPromise = null;
145
+ this.connectionCount = 0;
146
+ }
147
+ }
148
+ }
149
+
150
+ /**
151
+ * Force disconnect regardless of reference count
152
+ * @returns {Promise<void>}
153
+ */
154
+ async forceDisconnect() {
155
+ try {
156
+ if (mongoose.connection.readyState !== 0) {
157
+ await mongoose.disconnect();
158
+ if (!process.env.TUI_MODE) console.log('[MongooseHelper] ✅ Force disconnected from MongoDB');
159
+ }
160
+ } catch (error) {
161
+ console.error('[MongooseHelper] Force disconnect error:', error);
162
+ } finally {
163
+ this.isConnected = false;
164
+ this.connectionPromise = null;
165
+ this.connectionCount = 0;
166
+ }
167
+ }
168
+
169
+ /**
170
+ * Get connection status
171
+ * @returns {Object} Connection status info
172
+ */
173
+ getStatus() {
174
+ const readyStateMap = {
175
+ 0: 'disconnected',
176
+ 1: 'connected',
177
+ 2: 'connecting',
178
+ 3: 'disconnecting'
179
+ };
180
+
181
+ return {
182
+ isConnected: this.isConnected,
183
+ readyState: mongoose.connection.readyState,
184
+ readyStateText: readyStateMap[mongoose.connection.readyState] || 'unknown',
185
+ connectionCount: this.connectionCount,
186
+ host: mongoose.connection.host,
187
+ port: mongoose.connection.port,
188
+ name: mongoose.connection.name,
189
+ hasActiveConnection: mongoose.connection.readyState === 1
190
+ };
191
+ }
192
+
193
+ /**
194
+ * Execute function with automatic connection management
195
+ * @param {Function} fn - Async function to execute
196
+ * @param {Object} options - Options
197
+ * @returns {Promise<any>} Function result
198
+ */
199
+ async withConnection(fn, options = {}) {
200
+ await this.connect();
201
+
202
+ try {
203
+ const result = await fn(mongoose);
204
+ return result;
205
+ } finally {
206
+ if (options.autoDisconnect !== false) {
207
+ await this.disconnect();
208
+ }
209
+ }
210
+ }
211
+
212
+ /**
213
+ * Wait for connection to be ready
214
+ * @param {number} timeout - Timeout in milliseconds
215
+ * @returns {Promise<void>}
216
+ */
217
+ async waitForConnection(timeout = 10000) {
218
+ const startTime = Date.now();
219
+
220
+ while (mongoose.connection.readyState !== 1) {
221
+ if (Date.now() - startTime > timeout) {
222
+ throw new Error(`Connection timeout after ${timeout}ms`);
223
+ }
224
+
225
+ if (mongoose.connection.readyState === 0) {
226
+ throw new Error('Connection is disconnected');
227
+ }
228
+
229
+ await new Promise(resolve => setTimeout(resolve, 100));
230
+ }
231
+ }
232
+
233
+ /**
234
+ * Reset the helper state (useful for testing)
235
+ */
236
+ reset() {
237
+ this.connectionPromise = null;
238
+ this.isConnected = false;
239
+ this.connectionCount = 0;
240
+ this.connectionOptions = null;
241
+ }
242
+ }
243
+
244
+ // Singleton instance
245
+ const mongooseHelper = new MongooseHelper();
246
+
247
+ module.exports = {
248
+ MongooseHelper,
249
+ mongooseHelper,
250
+ connect: () => mongooseHelper.connect(),
251
+ disconnect: () => mongooseHelper.disconnect(),
252
+ forceDisconnect: () => mongooseHelper.forceDisconnect(),
253
+ withConnection: (fn, options) => mongooseHelper.withConnection(fn, options),
254
+ getStatus: () => mongooseHelper.getStatus(),
255
+ getMongoUri: () => mongooseHelper.getMongoUri(),
256
+ waitForConnection: (timeout) => mongooseHelper.waitForConnection(timeout),
257
+ reset: () => mongooseHelper.reset()
258
+ };
@@ -0,0 +1,230 @@
1
+ const { mongooseHelper } = require('./mongooseHelper');
2
+
3
+ /**
4
+ * Base class for scripts with database connectivity
5
+ * Provides automatic connection management and error handling
6
+ */
7
+ class ScriptBase {
8
+ constructor(options = {}) {
9
+ this.name = options.name || this.constructor.name;
10
+ this.autoDisconnect = options.autoDisconnect !== false;
11
+ this.timeout = options.timeout || 300000; // 5 minutes default
12
+ this.startTime = null;
13
+ this.context = null;
14
+ }
15
+
16
+ /**
17
+ * Main script execution method (to be implemented by subclasses)
18
+ * @param {Object} context - Execution context with mongoose instance
19
+ * @returns {Promise<any>} Script result
20
+ */
21
+ async execute(context) {
22
+ throw new Error('execute method must be implemented by subclass');
23
+ }
24
+
25
+ /**
26
+ * Setup method called before execution (optional override)
27
+ * @param {Object} context - Execution context
28
+ * @returns {Promise<void>}
29
+ */
30
+ async setup(context) {
31
+ // Override in subclasses if needed
32
+ }
33
+
34
+ /**
35
+ * Cleanup method called after execution (optional override)
36
+ * @param {Object} context - Execution context
37
+ * @returns {Promise<void>}
38
+ */
39
+ async cleanup(context) {
40
+ // Override in subclasses if needed
41
+ }
42
+
43
+ /**
44
+ * Run the script with automatic database connection management
45
+ * @returns {Promise<any>} Script result
46
+ */
47
+ async run() {
48
+ this.startTime = Date.now();
49
+
50
+ // Set up timeout handling
51
+ const timeoutPromise = new Promise((_, reject) => {
52
+ setTimeout(() => {
53
+ reject(new Error(`Script ${this.name} timed out after ${this.timeout}ms`));
54
+ }, this.timeout);
55
+ });
56
+
57
+ try {
58
+ if (!process.env.TUI_MODE) console.log(`[${this.name}] Starting script execution...`);
59
+
60
+ const executionPromise = this._executeWithConnection();
61
+ const result = await Promise.race([executionPromise, timeoutPromise]);
62
+
63
+ const duration = Date.now() - this.startTime;
64
+ if (!process.env.TUI_MODE) console.log(`[${this.name}] ✅ Completed in ${duration}ms`);
65
+
66
+ return result;
67
+ } catch (error) {
68
+ const duration = Date.now() - this.startTime;
69
+ console.error(`[${this.name}] ❌ Failed after ${duration}ms:`, error.message);
70
+
71
+ // Ensure cleanup on error
72
+ await this._handleError(error);
73
+ throw error;
74
+ }
75
+ }
76
+
77
+ /**
78
+ * Internal execution method with connection management
79
+ * @private
80
+ * @returns {Promise<any>}
81
+ */
82
+ async _executeWithConnection() {
83
+ return await mongooseHelper.withConnection(
84
+ async (mongoose) => {
85
+ // Create execution context
86
+ this.context = {
87
+ mongoose,
88
+ models: mongoose.models,
89
+ connection: mongoose.connection,
90
+ db: mongoose.connection.db,
91
+ script: {
92
+ name: this.name,
93
+ startTime: this.startTime,
94
+ timeout: this.timeout
95
+ }
96
+ };
97
+
98
+ // Call setup
99
+ await this.setup(this.context);
100
+
101
+ try {
102
+ // Execute main logic
103
+ const result = await this.execute(this.context);
104
+ return result;
105
+ } finally {
106
+ // Call cleanup
107
+ await this.cleanup(this.context);
108
+ }
109
+ },
110
+ { autoDisconnect: this.autoDisconnect }
111
+ );
112
+ }
113
+
114
+ /**
115
+ * Handle script errors and cleanup
116
+ * @private
117
+ * @param {Error} error - The error that occurred
118
+ */
119
+ async _handleError(error) {
120
+ try {
121
+ // Call cleanup with error context if available
122
+ if (this.context) {
123
+ await this.cleanup(this.context);
124
+ }
125
+ } catch (cleanupError) {
126
+ console.error(`[${this.name}] Cleanup error:`, cleanupError.message);
127
+ }
128
+
129
+ // Force disconnect on error
130
+ await mongooseHelper.forceDisconnect();
131
+ }
132
+
133
+ /**
134
+ * Get script execution status
135
+ * @returns {Object} Status information
136
+ */
137
+ getStatus() {
138
+ return {
139
+ name: this.name,
140
+ isRunning: this.startTime !== null,
141
+ startTime: this.startTime,
142
+ duration: this.startTime ? Date.now() - this.startTime : null,
143
+ timeout: this.timeout,
144
+ autoDisconnect: this.autoDisconnect,
145
+ connectionStatus: mongooseHelper.getStatus()
146
+ };
147
+ }
148
+
149
+ /**
150
+ * Validate script configuration
151
+ * @returns {Object} Validation result
152
+ */
153
+ validate() {
154
+ const errors = [];
155
+ const warnings = [];
156
+
157
+ // Check if execute method is implemented
158
+ if (this.execute === ScriptBase.prototype.execute) {
159
+ errors.push('execute method must be implemented');
160
+ }
161
+
162
+ // Check timeout value
163
+ if (this.timeout <= 0) {
164
+ errors.push('timeout must be greater than 0');
165
+ }
166
+
167
+ if (this.timeout > 3600000) { // 1 hour
168
+ warnings.push('timeout is very long (> 1 hour)');
169
+ }
170
+
171
+ // Check environment
172
+ if (!process.env.MONGODB_URI && !process.env.MONGO_URI) {
173
+ warnings.push('No MongoDB URI environment variable set, will use localhost');
174
+ }
175
+
176
+ return {
177
+ valid: errors.length === 0,
178
+ errors,
179
+ warnings
180
+ };
181
+ }
182
+
183
+ /**
184
+ * Create a child script with inherited configuration
185
+ * @param {Function} ChildScriptClass - Child script class
186
+ * @param {Object} options - Additional options for child
187
+ * @returns {ScriptBase} Child script instance
188
+ */
189
+ createChild(ChildScriptClass, options = {}) {
190
+ const childOptions = {
191
+ timeout: this.timeout,
192
+ autoDisconnect: this.autoDisconnect,
193
+ ...options
194
+ };
195
+
196
+ return new ChildScriptClass(childOptions);
197
+ }
198
+
199
+ /**
200
+ * Log script message with consistent formatting
201
+ * @param {string} level - Log level (info, warn, error, debug)
202
+ * @param {string} message - Message to log
203
+ * @param {any} data - Additional data to log
204
+ */
205
+ log(level, message, data = null) {
206
+ const timestamp = new Date().toISOString();
207
+ const prefix = `[${timestamp}][${this.name}]`;
208
+
209
+ switch (level) {
210
+ case 'info':
211
+ console.log(`${prefix} ${message}`, data || '');
212
+ break;
213
+ case 'warn':
214
+ console.warn(`${prefix} ⚠️ ${message}`, data || '');
215
+ break;
216
+ case 'error':
217
+ console.error(`${prefix} ❌ ${message}`, data || '');
218
+ break;
219
+ case 'debug':
220
+ if (process.env.DEBUG || process.env.NODE_ENV === 'development') {
221
+ console.debug(`${prefix} 🔍 ${message}`, data || '');
222
+ }
223
+ break;
224
+ default:
225
+ console.log(`${prefix} ${message}`, data || '');
226
+ }
227
+ }
228
+ }
229
+
230
+ module.exports = { ScriptBase };