@intranefr/superbackend 1.5.2 → 1.6.3

package/src/services/dataCleanup.service.js

@@ -0,0 +1,286 @@
+const mongoose = require('mongoose');
+
+function toSafeJsonError(error) {
+  const msg = error?.message || 'Operation failed';
+  const code = error?.code;
+  if (code === 'VALIDATION') return { status: 400, body: { error: msg } };
+  if (code === 'NOT_FOUND') return { status: 404, body: { error: msg } };
+  if (code === 'FORBIDDEN') return { status: 403, body: { error: msg } };
+  return { status: 500, body: { error: msg } };
+}
+
+function ensureDbConnection() {
+  const db = mongoose.connection?.db;
+  if (!db) {
+    throw Object.assign(new Error('MongoDB connection is not ready'), { code: 'FORBIDDEN' });
+  }
+  return db;
+}
+
+function normalizeCollectionName(value) {
+  const name = String(value || '').trim();
+  if (!name) throw Object.assign(new Error('collection is required'), { code: 'VALIDATION' });
+  if (!/^[A-Za-z0-9_.-]+$/.test(name)) {
+    throw Object.assign(new Error('Invalid collection name'), { code: 'VALIDATION' });
+  }
+  return name;
+}
+
+function normalizeDateField(value, fallback = 'createdAt') {
+  const field = String(value || fallback).trim();
+  if (!field) throw Object.assign(new Error('dateField is required'), { code: 'VALIDATION' });
+  if (field.includes('$')) {
+    throw Object.assign(new Error('Invalid dateField'), { code: 'VALIDATION' });
+  }
+  if (!/^[A-Za-z0-9_.-]+$/.test(field)) {
+    throw Object.assign(new Error('Invalid dateField'), { code: 'VALIDATION' });
+  }
+  return field;
+}
+
+function normalizeOlderThanDays(value) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) {
+    throw Object.assign(new Error('olderThanDays must be a positive number'), { code: 'VALIDATION' });
+  }
+  if (n > 36500) {
+    throw Object.assign(new Error('olderThanDays is too large'), { code: 'VALIDATION' });
+  }
+  return Math.floor(n);
+}
+
+function normalizeLimit(value, fallback = 5000) {
+  if (value === undefined || value === null || value === '') return fallback;
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) {
+    throw Object.assign(new Error('limit must be a positive number'), { code: 'VALIDATION' });
+  }
+  return Math.min(Math.floor(n), 50000);
+}
+
+async function getCollectionStatsByName(db, collection) {
+  const command = await db.command({ collStats: collection });
+  return {
+    name: collection,
+    ns: command?.ns || null,
+    count: Number(command?.count || 0),
+    sizeBytes: Number(command?.size || 0),
+    storageSizeBytes: Number(command?.storageSize || 0),
+    totalIndexSizeBytes: Number(command?.totalIndexSize || 0),
+    avgObjSizeBytes: Number(command?.avgObjSize || 0),
+  };
+}
+
+async function listCollectionStats() {
+  const db = ensureDbConnection();
+  const collections = await db.listCollections({}, { nameOnly: true }).toArray();
+  const names = (collections || []).map((c) => c.name).filter(Boolean).sort();
+
+  const stats = [];
+  for (const name of names) {
+    try {
+      // collStats can fail on internal collections depending on Mongo version/config.
+      const s = await getCollectionStatsByName(db, name);
+      stats.push(s);
+    } catch {
+      stats.push({
+        name,
+        ns: null,
+        count: 0,
+        sizeBytes: 0,
+        storageSizeBytes: 0,
+        totalIndexSizeBytes: 0,
+        avgObjSizeBytes: 0,
+        unavailable: true,
+      });
+    }
+  }
+
+  return stats;
+}
+
+async function getMongoGlobalStats() {
+  const db = ensureDbConnection();
+  const stats = await db.stats();
+  return {
+    db: stats?.db || null,
+    collections: Number(stats?.collections || 0),
+    views: Number(stats?.views || 0),
+    objects: Number(stats?.objects || 0),
+    dataSizeBytes: Number(stats?.dataSize || 0),
+    storageSizeBytes: Number(stats?.storageSize || 0),
+    indexes: Number(stats?.indexes || 0),
+    indexSizeBytes: Number(stats?.indexSize || 0),
+    totalSizeBytes: Number(stats?.totalSize || 0),
+  };
+}
+
+async function getOverviewStats() {
+  const [global, collections] = await Promise.all([
+    getMongoGlobalStats(),
+    listCollectionStats(),
+  ]);
+
+  return { global, collections };
+}
+
+async function ensureCollectionExists(db, collectionName) {
+  const collections = await db.listCollections({ name: collectionName }, { nameOnly: true }).toArray();
+  if (!collections || collections.length === 0) {
+    throw Object.assign(new Error('Collection not found'), { code: 'NOT_FOUND' });
+  }
+}
+
+function buildOlderThanQuery({ dateField, cutoff }) {
+  return {
+    [dateField]: {
+      $type: 'date',
+      $lt: cutoff,
+    },
+  };
+}
+
+function estimateReclaimableBytes({ candidateCount, collectionStats }) {
+  const avg = Number(collectionStats?.avgObjSizeBytes || 0);
+  if (avg > 0) return Math.round(candidateCount * avg);
+
+  const sizeBytes = Number(collectionStats?.sizeBytes || 0);
+  const count = Number(collectionStats?.count || 0);
+  if (sizeBytes > 0 && count > 0) {
+    return Math.round(candidateCount * (sizeBytes / count));
+  }
+
+  return 0;
+}
+
+async function dryRunCollectionCleanup({ collection, olderThanDays, dateField = 'createdAt' }) {
+  const db = ensureDbConnection();
+  const collectionName = normalizeCollectionName(collection);
+  const safeDateField = normalizeDateField(dateField);
+  const days = normalizeOlderThanDays(olderThanDays);
+
+  await ensureCollectionExists(db, collectionName);
+
+  const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
+  const query = buildOlderThanQuery({ dateField: safeDateField, cutoff });
+  const mongoCollection = db.collection(collectionName);
+
+  const [candidateCount, collectionStats] = await Promise.all([
+    mongoCollection.countDocuments(query),
+    getCollectionStatsByName(db, collectionName),
+  ]);
+
+  const estimatedReclaimableBytes = estimateReclaimableBytes({
+    candidateCount,
+    collectionStats,
+  });
+
+  return {
+    collection: collectionName,
+    dateField: safeDateField,
+    olderThanDays: days,
+    cutoffIso: cutoff.toISOString(),
+    candidateCount,
+    estimatedReclaimableBytes,
+    collectionStats,
+    notes: [
+      'Estimate is based on average object size and is not guaranteed to be reclaimed physically on disk immediately.',
+    ],
+  };
+}
+
+async function executeCollectionCleanup({
+  collection,
+  olderThanDays,
+  dateField = 'createdAt',
+  limit,
+  confirm,
+}) {
+  if (confirm !== true) {
+    throw Object.assign(new Error('Cleanup confirmation is required (confirm=true)'), { code: 'VALIDATION' });
+  }
+
+  const db = ensureDbConnection();
+  const collectionName = normalizeCollectionName(collection);
+  const safeDateField = normalizeDateField(dateField);
+  const days = normalizeOlderThanDays(olderThanDays);
+  const maxDelete = normalizeLimit(limit, 5000);
+
+  await ensureCollectionExists(db, collectionName);
+
+  const start = Date.now();
+  const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
+  const query = buildOlderThanQuery({ dateField: safeDateField, cutoff });
+  const mongoCollection = db.collection(collectionName);
+
+  const dryRun = await dryRunCollectionCleanup({
+    collection: collectionName,
+    olderThanDays: days,
+    dateField: safeDateField,
+  });
+
+  let remaining = maxDelete;
+  let deletedCount = 0;
+  const batchSize = 1000;
+
+  while (remaining > 0) {
+    const currentBatch = Math.min(batchSize, remaining);
+    const ids = await mongoCollection
+      .find(query, { projection: { _id: 1 } })
+      .sort({ _id: 1 })
+      .limit(currentBatch)
+      .toArray();
+
+    if (!ids.length) break;
+
+    const idList = ids.map((d) => d._id).filter(Boolean);
+    if (!idList.length) break;
+
+    const out = await mongoCollection.deleteMany({ _id: { $in: idList } });
+    const deleted = Number(out?.deletedCount || 0);
+    deletedCount += deleted;
+    remaining -= idList.length;
+
+    if (deleted === 0) break;
+  }
+
+  return {
+    collection: collectionName,
+    dateField: safeDateField,
+    olderThanDays: days,
+    cutoffIso: cutoff.toISOString(),
+    limitApplied: maxDelete,
+    dryRunCandidateCount: dryRun.candidateCount,
+    deletedCount,
+    estimatedReclaimableBytes: dryRun.estimatedReclaimableBytes,
+    durationMs: Date.now() - start,
+    notes: dryRun.notes,
+  };
+}
+
+async function inferCollectionFields(collectionName, sampleSize = 10) {
+  const db = ensureDbConnection();
+  const coll = db.collection(normalizeCollectionName(collectionName));
+  
+  const docs = await coll.find({}).limit(sampleSize).toArray();
+  const fieldSet = new Set();
+  
+  for (const doc of docs) {
+    const keys = Object.keys(doc);
+    for (const key of keys) {
+      fieldSet.add(key);
+    }
+  }
+  
+  return Array.from(fieldSet).sort();
+}
+
+module.exports = {
+  toSafeJsonError,
+  getMongoGlobalStats,
+  listCollectionStats,
+  getOverviewStats,
+  dryRunCollectionCleanup,
+  executeCollectionCleanup,
+  inferCollectionFields,
+};

package/src/services/jsonConfigs.service.js

@@ -2,6 +2,18 @@ const crypto = require('crypto');
 
 const JsonConfig = require('../models/JsonConfig');
 
+const logger = {
+  log: (...args) => {
+    if (process.env.DEBUG_JSON_CONFIGS === 'true' && !process.env.TUI_MODE) console.log(...args);
+  },
+  warn: (...args) => {
+    if (process.env.DEBUG_JSON_CONFIGS === 'true' && !process.env.TUI_MODE) console.warn(...args);
+  },
+  error: (...args) => {
+    console.error(...args);
+  }
+};
+
 const cache = new Map();
 
 function normalizeSlugBase(title) {
@@ -110,6 +122,108 @@ function clearAllJsonConfigCache() {
   cache.clear();
 }
 
+// Enhanced cache helpers
+function isJsonConfigCached(slug) {
+  const key = String(slug || '').trim();
+  if (!key) return false;
+  const entry = cache.get(key);
+  if (!entry) return false;
+  if (typeof entry.expiresAt === 'number' && Date.now() > entry.expiresAt) {
+    cache.delete(key);
+    return false;
+  }
+  return true;
+}
+
+function getJsonConfigCacheInfo(slug) {
+  const key = String(slug || '').trim();
+  if (!key) return null;
+  const entry = cache.get(key);
+  if (!entry) return { exists: false };
+  
+  const now = Date.now();
+  const isExpired = typeof entry.expiresAt === 'number' && now > entry.expiresAt;
+  if (isExpired) {
+    cache.delete(key);
+    return { exists: false };
+  }
+  
+  return {
+    exists: true,
+    expiresAt: entry.expiresAt,
+    ttlRemaining: entry.expiresAt ? Math.max(0, Math.floor((entry.expiresAt - now) / 1000)) : null,
+    size: JSON.stringify(entry.value).length
+  };
+}
+
+function clearJsonConfigCacheIfExists(slug) {
+  const key = String(slug || '').trim();
+  if (!key) return false;
+  const existed = cache.has(key);
+  cache.delete(key);
+  return existed;
+}
+
+function clearJsonConfigCacheIfExpired(slug) {
+  const info = getJsonConfigCacheInfo(slug);
+  if (!info || !info.exists) return false;
+  if (info.ttlRemaining !== null && info.ttlRemaining <= 0) {
+    cache.delete(String(slug));
+    return true;
+  }
+  return false;
+}
+
+function clearJsonConfigCacheBatch(slugs) {
+  if (!Array.isArray(slugs)) return 0;
+  let cleared = 0;
+  slugs.forEach(slug => {
+    if (clearJsonConfigCacheIfExists(slug)) cleared++;
+  });
+  return cleared;
+}
+
+function clearJsonConfigCacheByPattern(pattern) {
+  if (!pattern || typeof pattern !== 'string') return 0;
+  const regex = new RegExp(pattern.replace(/\*/g, '.*'));
+  let cleared = 0;
+  for (const key of cache.keys()) {
+    if (regex.test(key)) {
+      cache.delete(key);
+      cleared++;
+    }
+  }
+  return cleared;
+}
+
+function getJsonConfigCacheStats() {
+  const now = Date.now();
+  let entries = 0;
+  let expired = 0;
+  let totalSize = 0;
+  
+  for (const [key, entry] of cache.entries()) {
+    entries++;
+    if (typeof entry.expiresAt === 'number' && now > entry.expiresAt) {
+      expired++;
+    } else {
+      totalSize += JSON.stringify(entry.value).length;
+    }
+  }
+  
+  return {
+    totalEntries: entries,
+    expiredEntries: expired,
+    activeEntries: entries - expired,
+    totalSizeBytes: totalSize,
+    keys: Array.from(cache.keys())
+  };
+}
+
+function getJsonConfigCacheKeys() {
+  return Array.from(cache.keys());
+}
+
 async function listJsonConfigs() {
   return JsonConfig.find()
     .sort({ updatedAt: -1 })
@@ -122,7 +236,7 @@ async function getJsonConfigById(id) {
 }
 
 async function createJsonConfig({ title, jsonRaw, publicEnabled = false, cacheTtlSeconds = 0, alias }) {
-  console.log('createJsonConfig called with:', { title, jsonRaw, publicEnabled, cacheTtlSeconds, alias });
+  logger.log('createJsonConfig called with:', { title, jsonRaw, publicEnabled, cacheTtlSeconds, alias });
   
   const normalizedTitle = String(title || '').trim();
   if (!normalizedTitle) {
@@ -142,7 +256,7 @@ async function createJsonConfig({ title, jsonRaw, publicEnabled = false, cacheTt
   let normalizedAlias = null;
   if (alias !== undefined && alias !== null) {
     normalizedAlias = normalizeAlias(alias);
-    console.log('Normalized alias:', normalizedAlias);
+    logger.log('Normalized alias:', normalizedAlias);
     if (normalizedAlias && !(await validateAliasUniqueness(normalizedAlias))) {
       const err = new Error('Alias must be unique and not conflict with existing slugs or aliases');
       err.code = 'ALIAS_NOT_UNIQUE';
@@ -175,7 +289,7 @@ async function createJsonConfig({ title, jsonRaw, publicEnabled = false, cacheTt
 }
 
 async function updateJsonConfig(id, patch) {
-  console.log('updateJsonConfig called with id:', id, 'patch:', patch);
+  logger.log('updateJsonConfig called with id:', id, 'patch:', patch);
   
   const doc = await JsonConfig.findById(id);
   if (!doc) {
@@ -184,7 +298,7 @@ async function updateJsonConfig(id, patch) {
     throw err;
   }
 
-  console.log('Found document:', doc.toObject());
+  logger.log('Found document:', doc.toObject());
 
   const oldSlug = doc.slug;
   const oldAlias = doc.alias;
@@ -222,14 +336,14 @@ async function updateJsonConfig(id, patch) {
 
   if (patch && Object.prototype.hasOwnProperty.call(patch, 'alias')) {
     const newAlias = patch.alias;
-    console.log('Processing alias update. newAlias:', newAlias);
+    logger.log('Processing alias update. newAlias:', newAlias);
     
     if (newAlias === null || newAlias === undefined || newAlias === '') {
       doc.alias = undefined;
-      console.log('Setting alias to undefined');
+      logger.log('Setting alias to undefined');
     } else {
       const normalizedAlias = normalizeAlias(newAlias);
-      console.log('Normalized alias for update:', normalizedAlias);
+      logger.log('Normalized alias for update:', normalizedAlias);
       
       if (!normalizedAlias) {
         const err = new Error('Invalid alias format');
@@ -244,7 +358,7 @@ async function updateJsonConfig(id, patch) {
       }
       
       doc.alias = normalizedAlias;
-      console.log('Setting alias to:', normalizedAlias);
+      logger.log('Setting alias to:', normalizedAlias);
     }
   }
 
@@ -252,9 +366,9 @@ async function updateJsonConfig(id, patch) {
     doc.slug = await generateUniqueSlugFromTitle(doc.title);
   }
 
-  console.log('Document before save:', doc.toObject());
+  logger.log('Document before save:', doc.toObject());
   await doc.save();
-  console.log('Document after save:', doc.toObject());
+  logger.log('Document after save:', doc.toObject());
 
   clearJsonConfigCache(oldSlug);
   clearJsonConfigCache(doc.slug);
@@ -374,16 +488,176 @@ async function getJsonConfigPublicPayload(slug, { raw = false } = {}) {
   };
 }
 
+// Cache-aware update helpers
+async function updateJsonConfigWithCacheInvalidation(id, patch) {
+  logger.log('updateJsonConfigWithCacheInvalidation called with id:', id, 'patch:', patch);
+  
+  const doc = await JsonConfig.findById(id);
+  if (!doc) {
+    const err = new Error('JSON config not found');
+    err.code = 'NOT_FOUND';
+    throw err;
+  }
+
+  const oldSlug = doc.slug;
+  const oldAlias = doc.alias;
+
+  // Apply updates using existing update logic
+  if (patch && Object.prototype.hasOwnProperty.call(patch, 'title')) {
+    const title = String(patch.title || '').trim();
+    if (!title) {
+      const err = new Error('title is required');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+    doc.title = title;
+  }
+
+  if (patch && Object.prototype.hasOwnProperty.call(patch, 'publicEnabled')) {
+    doc.publicEnabled = Boolean(patch.publicEnabled);
+  }
+
+  if (patch && Object.prototype.hasOwnProperty.call(patch, 'cacheTtlSeconds')) {
+    const ttl = Number(patch.cacheTtlSeconds || 0);
+    doc.cacheTtlSeconds = Number.isNaN(ttl) ? 0 : Math.max(0, ttl);
+  }
+
+  if (patch && Object.prototype.hasOwnProperty.call(patch, 'jsonRaw')) {
+    if (patch.jsonRaw === null || patch.jsonRaw === undefined) {
+      const err = new Error('jsonRaw is required');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+
+    parseJsonOrThrow(patch.jsonRaw);
+    doc.jsonRaw = String(patch.jsonRaw);
+    doc.jsonHash = computeJsonHash(doc.jsonRaw);
+  }
+
+  if (patch && Object.prototype.hasOwnProperty.call(patch, 'alias')) {
+    const newAlias = patch.alias;
+    
+    if (newAlias === null || newAlias === undefined || newAlias === '') {
+      doc.alias = undefined;
+    } else {
+      const normalizedAlias = normalizeAlias(newAlias);
+      
+      if (!normalizedAlias) {
+        const err = new Error('Invalid alias format');
+        err.code = 'VALIDATION';
+        throw err;
+      }
+
+      if (!(await validateAliasUniqueness(normalizedAlias, doc._id))) {
+        const err = new Error('Alias must be unique and not conflict with existing slugs or aliases');
+        err.code = 'ALIAS_NOT_UNIQUE';
+        throw err;
+      }
+
+      doc.alias = normalizedAlias;
+    }
+  }
+
+  await doc.save();
+
+  // Clear all related cache entries
+  clearJsonConfigCache(oldSlug);
+  clearJsonConfigCache(doc.slug);
+  if (oldAlias) {
+    clearJsonConfigCache(oldAlias);
+  }
+  if (doc.alias) {
+    clearJsonConfigCache(doc.alias);
+  }
+
+  return doc.toObject();
+}
+
+async function updateJsonConfigValueBySlug(slug, updateFn, options = {}) {
+  const key = String(slug || '').trim();
+  if (!key) {
+    const err = new Error('slug is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  if (typeof updateFn !== 'function') {
+    const err = new Error('updateFn must be a function');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  // Get current value (bypass cache to ensure fresh data)
+  const currentValue = await getJsonConfigValueBySlug(key, { bypassCache: true });
+  
+  // Apply update function
+  let newValue;
+  try {
+    newValue = await updateFn(currentValue);
+  } catch (error) {
+    const err = new Error(`Update function failed: ${error.message}`);
+    err.code = 'UPDATE_FAILED';
+    err.cause = error;
+    throw err;
+  }
+
+  // Validate new value
+  const jsonRaw = JSON.stringify(newValue);
+  parseJsonOrThrow(jsonRaw); // This will throw if invalid JSON
+
+  // Find the document
+  const doc = await JsonConfig.findOne({
+    $or: [
+      { slug: key },
+      { alias: key }
+    ]
+  });
+
+  if (!doc) {
+    const err = new Error('JSON config not found');
+    err.code = 'NOT_FOUND';
+    throw err;
+  }
+
+  // Update the document
+  doc.jsonRaw = jsonRaw;
+  doc.jsonHash = computeJsonHash(jsonRaw);
+  await doc.save();
+
+  // Clear cache if requested (default: true)
+  if (options.invalidateCache !== false) {
+    clearJsonConfigCache(doc.slug);
+    if (doc.alias) {
+      clearJsonConfigCache(doc.alias);
+    }
+    clearJsonConfigCache(key);
+  }
+
+  return newValue;
+}
+
 module.exports = {
   normalizeSlugBase,
   generateUniqueSlugFromTitle,
   parseJsonOrThrow,
   clearJsonConfigCache,
   clearAllJsonConfigCache,
+  // Enhanced cache helpers
+  isJsonConfigCached,
+  getJsonConfigCacheInfo,
+  clearJsonConfigCacheIfExists,
+  clearJsonConfigCacheIfExpired,
+  clearJsonConfigCacheBatch,
+  clearJsonConfigCacheByPattern,
+  getJsonConfigCacheStats,
+  getJsonConfigCacheKeys,
+  // Core functions
   listJsonConfigs,
   getJsonConfigById,
   createJsonConfig,
   updateJsonConfig,
+  updateJsonConfigWithCacheInvalidation,
+  updateJsonConfigValueBySlug,
   regenerateJsonConfigSlug,
   deleteJsonConfig,
   getJsonConfig: getJsonConfigValueBySlug,