@intranefr/superbackend 1.4.3 → 1.5.0

package/src/services/consoleOverride.service.js

@@ -0,0 +1,291 @@
+const fs = require('fs');
+const path = require('path');
+
+let originalConsole = null;
+let logFileStream = null;
+let isActive = false;
+let isWriting = false;
+let memoryInterval = null;
+let logLines = [];
+const MAX_LOG_LINES = 2000;
+
+// Store the truly original console methods at module load time
+const TRULY_ORIGINAL_CONSOLE = {
+  log: console.log,
+  error: console.error,
+  warn: console.warn,
+  info: console.info,
+  debug: console.debug
+};
+
+/**
+ * Console Override Service
+ * Provides dual logging to stdout and file in non-production environments
+ */
+const consoleOverride = {
+  /**
+   * Initialize console override
+   * @param {Object} options - Configuration options
+   * @param {string} options.logFile - Log file path (default: 'stdout.log')
+   * @param {boolean} options.forceEnable - Force enable regardless of NODE_ENV
+   */
+  init(options = {}) {
+    if (isActive) {
+      return; // Already initialized
+    }
+
+    const nodeEnv = process.env.NODE_ENV || 'development';
+    const forceEnabled = options.forceEnable || process.env.CONSOLE_OVERRIDE_ENABLED === 'true';
+    const forceDisabled = process.env.CONSOLE_OVERRIDE_ENABLED === 'false';
+
+    // Skip if production and not force enabled, or if force disabled
+    if ((nodeEnv === 'production' && !forceEnabled) || forceDisabled) {
+      return;
+    }
+
+    const logFile = options.logFile || process.env.CONSOLE_LOG_FILE || 'stdout.log';
+    const logPath = path.resolve(process.cwd(), logFile);
+
+    try {
+      // Close any existing stream before truncating
+      if (logFileStream && !logFileStream.destroyed) {
+        logFileStream.end();
+        logFileStream = null;
+      }
+      
+      // Wait a bit for stream to fully close, then truncate
+      setTimeout(() => {
+        // Truncate log file on initialization (start with empty file)
+        if (fs.existsSync(logPath)) {
+          fs.truncateSync(logPath, 0);
+        }
+        
+        // Create file stream for appending with error handling
+        logFileStream = fs.createWriteStream(logPath, { flags: 'a' });
+        
+        // Handle stream errors
+        logFileStream.on('error', (error) => {
+          if (originalConsole && originalConsole.error && !isWriting) {
+            originalConsole.error('❌ Log stream error:', error.message);
+          }
+          isActive = false;
+        });
+        
+        // Store original console
+        originalConsole = { ...console };
+        
+        // Override console methods
+        this._overrideConsoleMethods();
+        
+        // Start memory management interval (1 minute)
+        this._startMemoryManagement();
+        
+        isActive = true;
+        
+        // Log initialization using original console to avoid recursion
+        const initMsg = `📝 Console override initialized - logging to ${logPath}`;
+        originalConsole.log(initMsg);
+        this._writeToFile(initMsg);
+      }, 10);
+      
+    } catch (error) {
+      // Fallback to console-only logging
+      originalConsole = originalConsole || console;
+      originalConsole.error('❌ Console override failed:', error.message);
+      isActive = false;
+    }
+  },
+
+  /**
+   * Override individual console methods
+   * @private
+   */
+  _overrideConsoleMethods() {
+    const methods = ['log', 'error', 'warn', 'info', 'debug'];
+    
+    methods.forEach(method => {
+      console[method] = (...args) => {
+        // Call the truly original console method
+        TRULY_ORIGINAL_CONSOLE[method](...args);
+        
+        // Write to file if stream is available and not already writing
+        if (logFileStream && !logFileStream.destroyed && !isWriting) {
+          this._writeToFile(args);
+        }
+      };
+    });
+  },
+
+  /**
+   * Write message to file
+   * @param {string|Array} message - Message to write
+   * @private
+   */
+  _writeToFile(message) {
+    if (!logFileStream || logFileStream.destroyed) {
+      return;
+    }
+
+    isWriting = true;
+    try {
+      const messageStr = Array.isArray(message) 
+        ? message.map(arg => typeof arg === 'object' ? JSON.stringify(arg, null, 2) : String(arg)).join(' ')
+        : String(message);
+      
+      // Add to memory buffer
+      logLines.push(messageStr);
+      
+      // Only write if stream is available and not in the middle of a rewrite
+      if (logFileStream && !logFileStream.destroyed) {
+        logFileStream.write(messageStr + '\n');
+      }
+    } catch (writeError) {
+      // Prevent infinite recursion - use original console for errors
+      if (originalConsole && originalConsole.error && !isWriting) {
+        originalConsole.error('❌ Log write error:', writeError.message);
+      }
+    } finally {
+      isWriting = false;
+    }
+  },
+
+  /**
+   * Start memory management interval
+   * @private
+   */
+  _startMemoryManagement() {
+    // Clear any existing interval
+    if (memoryInterval) {
+      clearInterval(memoryInterval);
+    }
+    
+    // Set up 1-minute interval to manage log lines
+    memoryInterval = setInterval(() => {
+      this._manageLogMemory();
+    }, 60000); // 1 minute
+  },
+
+  /**
+   * Manage log memory by keeping only last MAX_LOG_LINES
+   * @private
+   */
+  _manageLogMemory() {
+    if (logLines.length > MAX_LOG_LINES) {
+      // Keep only the last MAX_LOG_LINES
+      const excessLines = logLines.length - MAX_LOG_LINES;
+      logLines = logLines.slice(excessLines);
+      
+      // Rewrite the log file with only the recent lines
+      this._rewriteLogFile();
+    }
+  },
+
+  /**
+   * Rewrite log file with current memory buffer
+   * @private
+   */
+  _rewriteLogFile() {
+    if (!logFileStream || !logFileStream.path) {
+      return;
+    }
+
+    try {
+      const logPath = logFileStream.path;
+      
+      // Write recent lines to file (this truncates the file)
+      const fileContent = logLines.join('\n') + '\n';
+      fs.writeFileSync(logPath, fileContent, { flag: 'w' });
+      
+      // Reopen stream for appending
+      logFileStream = fs.createWriteStream(logPath, { flags: 'a' });
+      
+      // Reattach error handler
+      logFileStream.on('error', (error) => {
+        if (originalConsole && originalConsole.error && !isWriting) {
+          originalConsole.error('❌ Log stream error:', error.message);
+        }
+      });
+      
+    } catch (error) {
+      if (originalConsole && originalConsole.error) {
+        originalConsole.error('❌ Log file rewrite error:', error.message);
+      }
+    }
+  },
+
+  /**
+   * Restore original console
+   */
+  restore() {
+    if (!isActive && !originalConsole) {
+      return;
+    }
+
+    // Clear memory management interval
+    if (memoryInterval) {
+      clearInterval(memoryInterval);
+      memoryInterval = null;
+    }
+
+    // Restore original console methods using the truly original ones
+    ['log', 'error', 'warn', 'info', 'debug'].forEach(method => {
+      console[method] = TRULY_ORIGINAL_CONSOLE[method];
+    });
+
+    // Close file stream
+    if (logFileStream && !logFileStream.destroyed) {
+      logFileStream.end();
+    }
+
+    // Reset state
+    isActive = false;
+    originalConsole = null;
+    logFileStream = null;
+    logLines = [];
+    isWriting = false;
+  },
+
+  /**
+   * Check if override is active
+   * @returns {boolean}
+   */
+  isActive() {
+    return isActive;
+  },
+
+  /**
+   * Get current memory lines count (for testing/debugging)
+   * @returns {number}
+   */
+  getMemoryLinesCount() {
+    return logLines.length;
+  },
+
+  /**
+   * Get current log path
+   * @returns {string|null}
+   */
+  getLogPath() {
+    if (!logFileStream) {
+      return null;
+    }
+    return logFileStream.path;
+  }
+};
+
+// Cleanup on process exit
+process.on('exit', () => {
+  consoleOverride.restore();
+});
+
+process.on('SIGINT', () => {
+  consoleOverride.restore();
+  process.exit(0);
+});
+
+process.on('SIGTERM', () => {
+  consoleOverride.restore();
+  process.exit(0);
+});
+
+module.exports = consoleOverride;

package/src/services/email.service.js

@@ -74,7 +74,7 @@ const sendEmail = async ({
     from ||
     (await getSetting(
       "EMAIL_FROM",
-      process.env.EMAIL_FROM || "SaaSBackend <no-reply@resend.dev>",
+      process.env.EMAIL_FROM || "SuperBackend <no-reply@resend.dev>",
     ));
   const toArray = Array.isArray(to) ? to : [to];
 
@@ -338,6 +338,20 @@ const replaceTemplateVariables = (template, variables) => {
   return result;
 };
 
+// Helper to clear cache (for testing)
+const clearCache = () => {
+  settingsCache.clear();
+  resendClient = null;
+};
+
+// Helper to clear cache and reinitialize (for testing)
+const clearCacheAndReinitialize = async () => {
+  settingsCache.clear();
+  resendClient = null;
+  // Reinitialize with current env vars
+  await initResend();
+};
+
 module.exports = {
   sendEmail,
   sendPasswordResetEmail,
@@ -348,4 +362,6 @@ module.exports = {
   sendSubscriptionEmail,
   sendWaitingListEmail,
   replaceTemplateVariables,
+  clearCache,
+  clearCacheAndReinitialize,
 };

package/src/services/headlessExternalModels.service.js

@@ -0,0 +1,292 @@
+const mongoose = require('mongoose');
+
+const HeadlessModelDefinition = require('../models/HeadlessModelDefinition');
+const { normalizeCodeIdentifier, computeSchemaHash } = require('./headlessModels.service');
+
+function isObjectId(value) {
+  if (!value) return false;
+  if (value instanceof mongoose.Types.ObjectId) return true;
+  if (value && typeof value === 'object' && value._bsontype === 'ObjectID') return true;
+  return false;
+}
+
+function detectFieldType(value) {
+  if (value === null || value === undefined) return null;
+  if (isObjectId(value)) return 'objectid';
+  if (value instanceof Date) return 'date';
+  if (Array.isArray(value)) return 'array';
+  const t = typeof value;
+  if (t === 'string') return 'string';
+  if (t === 'number') return 'number';
+  if (t === 'boolean') return 'boolean';
+  if (t === 'object') return 'object';
+  return null;
+}
+
+function pickBestType(seenTypes) {
+  const types = new Set(Array.from(seenTypes || []).filter(Boolean));
+  if (types.size === 0) return { type: 'object', warning: 'No non-null sample values' };
+  if (types.size === 1) return { type: Array.from(types)[0], warning: null };
+  if (types.has('array')) return { type: 'array', warning: `Mixed types: ${Array.from(types).join('|')}` };
+  if (types.has('object')) return { type: 'object', warning: `Mixed types: ${Array.from(types).join('|')}` };
+  return { type: 'string', warning: `Mixed types: ${Array.from(types).join('|')}` };
+}
+
+function tryInferRefFromFieldName(fieldName, externalModelsByCollection) {
+  const name = String(fieldName || '').trim();
+  if (!name) return null;
+
+  const lower = name.toLowerCase();
+  if (!lower.endsWith('id') || lower === 'id') return null;
+
+  const stem = name.slice(0, -2);
+  const candidates = [stem, `${stem}s`, `${stem}es`];
+  for (const c of candidates) {
+    const match = externalModelsByCollection.get(String(c).toLowerCase());
+    if (match) return match;
+  }
+  return null;
+}
+
+function normalizeIndexFromMongo(idx) {
+  if (!idx || typeof idx !== 'object') return null;
+  const fields = idx.key;
+  if (!fields || typeof fields !== 'object') return null;
+
+  const options = { ...idx };
+  delete options.key;
+  delete options.v;
+  delete options.ns;
+
+  return { fields, options };
+}
+
+async function listExternalCollections({ q, includeSystem } = {}) {
+  if (!mongoose.connection || !mongoose.connection.db) {
+    const err = new Error('Mongo connection not ready');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const filter = {};
+  if (q) filter.name = { $regex: String(q), $options: 'i' };
+
+  const cursor = await mongoose.connection.db.listCollections(filter, { nameOnly: true });
+  const items = await cursor.toArray();
+
+  const out = [];
+  for (const c of items) {
+    const name = String(c && c.name ? c.name : '').trim();
+    if (!name) continue;
+    if (!includeSystem && name.startsWith('system.')) continue;
+    out.push({ name, type: 'collection' });
+  }
+
+  out.sort((a, b) => a.name.localeCompare(b.name));
+  return out;
+}
+
+async function inferExternalModelFromCollection({ collectionName, sampleSize = 200 } = {}) {
+  if (!mongoose.connection || !mongoose.connection.db) {
+    const err = new Error('Mongo connection not ready');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const collName = String(collectionName || '').trim();
+  if (!collName) {
+    const err = new Error('collectionName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const N = Math.max(1, Math.min(Number(sampleSize) || 200, 1000));
+  const coll = mongoose.connection.db.collection(collName);
+
+  let docs;
+  try {
+    docs = await coll.aggregate([{ $sample: { size: N } }]).toArray();
+  } catch {
+    docs = await coll.find({}).limit(N).toArray();
+  }
+
+  const externalModels = await HeadlessModelDefinition.find({ isActive: true, sourceType: 'external' })
+    .select({ codeIdentifier: 1, sourceCollectionName: 1 })
+    .lean();
+
+  const externalModelsByCollection = new Map();
+  for (const m of externalModels || []) {
+    const cn = String(m?.sourceCollectionName || '').trim();
+    const code = String(m?.codeIdentifier || '').trim();
+    if (cn && code) externalModelsByCollection.set(cn.toLowerCase(), code);
+  }
+
+  const perField = new Map();
+  const warnings = [];
+
+  for (const doc of docs || []) {
+    if (!doc || typeof doc !== 'object') continue;
+    for (const [k, v] of Object.entries(doc)) {
+      if (!k || k === '_id') continue;
+      const type = detectFieldType(v);
+      let stats = perField.get(k);
+      if (!stats) {
+        stats = { seenTypes: new Set(), objectIdCount: 0, nonNullCount: 0 };
+        perField.set(k, stats);
+      }
+      if (type) stats.seenTypes.add(type);
+      if (type === 'objectid') stats.objectIdCount += 1;
+      if (v !== null && v !== undefined) stats.nonNullCount += 1;
+    }
+  }
+
+  const fields = [];
+  for (const [name, stats] of perField.entries()) {
+    const { type, warning } = pickBestType(stats.seenTypes);
+
+    if (warning) warnings.push(`Field ${name}: ${warning}`);
+
+    if (type === 'objectid') {
+      const refModelCode = tryInferRefFromFieldName(name, externalModelsByCollection);
+      if (refModelCode) {
+        fields.push({ name, type: 'ref', required: false, unique: false, refModelCode });
+      } else {
+        fields.push({ name, type: 'string', required: false, unique: false });
+      }
+      continue;
+    }
+
+    if (type === 'date') {
+      fields.push({ name, type: 'date', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'array') {
+      fields.push({ name, type: 'array', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'object') {
+      fields.push({ name, type: 'object', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'number' || type === 'boolean' || type === 'string') {
+      fields.push({ name, type, required: false, unique: false });
+      continue;
+    }
+
+    fields.push({ name, type: 'object', required: false, unique: false });
+  }
+
+  fields.sort((a, b) => a.name.localeCompare(b.name));
+
+  let indexes = [];
+  try {
+    const idx = await coll.indexes();
+    indexes = (idx || []).map(normalizeIndexFromMongo).filter(Boolean);
+  } catch {
+    indexes = [];
+  }
+
+  const fieldsHash = computeSchemaHash({ fields, indexes });
+
+  return {
+    collectionName: collName,
+    fields,
+    indexes,
+    warnings,
+    stats: {
+      sampled: (docs || []).length,
+      maxSampleSize: N,
+      fields: fields.length,
+    },
+    fieldsHash,
+  };
+}
+
+async function createOrUpdateExternalModel({ collectionName, codeIdentifier, displayName, sampleSize } = {}) {
+  const cn = String(collectionName || '').trim();
+  if (!cn) {
+    const err = new Error('collectionName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const code = normalizeCodeIdentifier(codeIdentifier);
+  if (!code.startsWith('ext_')) {
+    const err = new Error('External model codeIdentifier must start with ext_');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const name = String(displayName || code).trim();
+  if (!name) {
+    const err = new Error('displayName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const inferred = await inferExternalModelFromCollection({ collectionName: cn, sampleSize });
+
+  const existing = await HeadlessModelDefinition.findOne({ codeIdentifier: code, isActive: true });
+  if (!existing) {
+    const doc = await HeadlessModelDefinition.create({
+      codeIdentifier: code,
+      displayName: name,
+      description: '',
+      fields: inferred.fields,
+      indexes: inferred.indexes,
+      fieldsHash: inferred.fieldsHash,
+      version: 1,
+      previousFields: [],
+      previousIndexes: [],
+      sourceType: 'external',
+      sourceCollectionName: cn,
+      isExternal: true,
+      inference: {
+        enabled: true,
+        lastInferredAt: new Date(),
+        sampleSize: Number(sampleSize) || null,
+        warnings: inferred.warnings || [],
+        stats: inferred.stats || null,
+      },
+      isActive: true,
+    });
+
+    return { created: true, item: doc.toObject(), inference: inferred };
+  }
+
+  existing.displayName = name;
+  existing.sourceType = 'external';
+  existing.sourceCollectionName = cn;
+  existing.isExternal = true;
+
+  const newHash = inferred.fieldsHash;
+  if (newHash !== existing.fieldsHash) {
+    existing.previousFields = existing.fields;
+    existing.previousIndexes = existing.indexes;
+    existing.fields = inferred.fields;
+    existing.indexes = inferred.indexes;
+    existing.fieldsHash = newHash;
+    existing.version = Number(existing.version || 1) + 1;
+  }
+
+  existing.inference = {
+    enabled: true,
+    lastInferredAt: new Date(),
+    sampleSize: Number(sampleSize) || null,
+    warnings: inferred.warnings || [],
+    stats: inferred.stats || null,
+  };
+
+  await existing.save();
+
+  return { created: false, item: existing.toObject(), inference: inferred };
+}
+
+module.exports = {
+  listExternalCollections,
+  inferExternalModelFromCollection,
+  createOrUpdateExternalModel,
+};

package/src/services/headlessModels.service.js

@@ -89,6 +89,18 @@ function getMongoCollectionName(codeIdentifier) {
   return `${MODEL_COLLECTION_PREFIX}${code}`;
 }
 
+function isExternalDefinition(def) {
+  return def && (def.sourceType === 'external' || def.isExternal === true);
+}
+
+function getCollectionNameForDefinition(def) {
+  if (isExternalDefinition(def)) {
+    const cn = String(def.sourceCollectionName || '').trim();
+    return cn || getMongoCollectionName(def.codeIdentifier);
+  }
+  return getMongoCollectionName(def.codeIdentifier);
+}
+
 function buildSchemaFromDefinition(def) {
   const schemaShape = {};
   for (const field of def.fields || []) {
@@ -99,12 +111,14 @@ function buildSchemaFromDefinition(def) {
     schemaShape[fieldName] = toMongooseField(field);
   }
 
-  schemaShape._headlessModelCode = { type: String, default: def.codeIdentifier, index: true };
-  schemaShape._headlessSchemaVersion = { type: Number, default: def.version, index: true };
+  if (!isExternalDefinition(def)) {
+    schemaShape._headlessModelCode = { type: String, default: def.codeIdentifier, index: true };
+    schemaShape._headlessSchemaVersion = { type: Number, default: def.version, index: true };
+  }
 
   const schema = new mongoose.Schema(schemaShape, {
     timestamps: true,
-    collection: getMongoCollectionName(def.codeIdentifier),
+    collection: getCollectionNameForDefinition(def),
     strict: false,
   });
 
@@ -254,6 +268,8 @@ async function disableModelDefinition(codeIdentifier) {
 const modelCache = new Map();
 
 async function ensureAutoMigration(modelDef) {
+  if (isExternalDefinition(modelDef)) return;
+
   const collectionName = getMongoCollectionName(modelDef.codeIdentifier);
   const coll = mongoose.connection.collection(collectionName);
 
@@ -318,7 +334,8 @@ async function getDynamicModel(codeIdentifier) {
     throw err;
   }
 
-  const cacheKey = `${def.codeIdentifier}:${def.version}:${def.fieldsHash}`;
+  const collectionName = getCollectionNameForDefinition(def);
+  const cacheKey = `${def.codeIdentifier}:${def.version}:${def.fieldsHash}:${collectionName}`;
   const cached = modelCache.get(cacheKey);
   if (cached) return cached;
 
@@ -334,8 +351,10 @@ async function getDynamicModel(codeIdentifier) {
   const schema = buildSchemaFromDefinition(def);
   const Model = mongoose.model(modelName, schema);
 
-  await ensureAutoMigration(def);
-  await ensureIndexesBestEffort(Model);
+  if (!isExternalDefinition(def)) {
+    await ensureAutoMigration(def);
+    await ensureIndexesBestEffort(Model);
+  }
 
   modelCache.set(cacheKey, Model);
   return Model;
@@ -346,6 +365,7 @@ module.exports = {
   normalizeCodeIdentifier,
   getMongooseModelName,
   getMongoCollectionName,
+  getCollectionNameForDefinition,
   computeSchemaHash,
   listModelDefinitions,
   getModelDefinitionByCode,