@intranefr/superbackend 1.4.4 → 1.5.1

package/src/services/fileManagerStoragePolicy.service.js

@@ -0,0 +1,285 @@
+const mongoose = require('mongoose');
+
+const Asset = require('../models/Asset');
+const FileEntry = require('../models/FileEntry');
+const RbacGroup = require('../models/RbacGroup');
+const RbacGroupMember = require('../models/RbacGroupMember');
+
+const globalSettingsService = require('./globalSettings.service');
+
+const SETTING_POLICY_JSON = 'FILE_MANAGER_STORAGE_POLICY_JSON';
+const SETTING_LEGACY_MAX_UPLOAD = 'FILE_MANAGER_MAX_UPLOAD_BYTES';
+
+const DEFAULT_MAX_UPLOAD_BYTES = 1073741824;
+const DEFAULT_MAX_STORAGE_BYTES = 104857600;
+
+function normalizeObjectId(id, name) {
+  const str = String(id || '');
+  if (!mongoose.Types.ObjectId.isValid(str)) {
+    const err = new Error(`${name} must be a valid ObjectId`);
+    err.code = 'VALIDATION';
+    throw err;
+  }
+  return new mongoose.Types.ObjectId(str);
+}
+
+function normalizeDriveType(value) {
+  const t = String(value || '').trim();
+  if (t === 'user' || t === 'group' || t === 'org') return t;
+  const err = new Error('driveType must be one of: user, group, org');
+  err.code = 'VALIDATION';
+  throw err;
+}
+
+function toPositiveIntOrNull(value) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  return Math.floor(n);
+}
+
+async function loadPolicy() {
+  const raw = await globalSettingsService.getSettingValue(SETTING_POLICY_JSON, null);
+  if (!raw) return { version: 1, global: {}, orgs: {} };
+
+  try {
+    const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw;
+    if (!parsed || typeof parsed !== 'object') return { version: 1, global: {}, orgs: {} };
+
+    const version = toPositiveIntOrNull(parsed.version) || 1;
+    const global = parsed.global && typeof parsed.global === 'object' ? parsed.global : {};
+    const orgs = parsed.orgs && typeof parsed.orgs === 'object' ? parsed.orgs : {};
+
+    return { version, global, orgs };
+  } catch {
+    return { version: 1, global: {}, orgs: {} };
+  }
+}
+
+function getEnvDefaultMaxUploadBytes() {
+  const fromEnv = toPositiveIntOrNull(process.env.FILE_MANAGER_DEFAULT_MAX_UPLOAD_BYTES);
+  return fromEnv || DEFAULT_MAX_UPLOAD_BYTES;
+}
+
+async function getDefaultMaxUploadBytes() {
+  const legacy = await globalSettingsService.getSettingValue(SETTING_LEGACY_MAX_UPLOAD, null);
+  const legacyParsed = toPositiveIntOrNull(legacy);
+  return legacyParsed || getEnvDefaultMaxUploadBytes();
+}
+
+function getDefaultMaxStorageBytes() {
+  const fromEnv = toPositiveIntOrNull(process.env.FILE_MANAGER_DEFAULT_MAX_STORAGE_BYTES);
+  return fromEnv || DEFAULT_MAX_STORAGE_BYTES;
+}
+
+async function getUserOrgGroupIds({ userId, orgId }) {
+  const uid = normalizeObjectId(userId, 'userId');
+  const oid = normalizeObjectId(orgId, 'orgId');
+
+  const links = await RbacGroupMember.find({ userId: uid }).select('groupId').lean();
+  const groupIds = links.map((l) => l.groupId).filter(Boolean);
+  if (!groupIds.length) return [];
+
+  const groups = await RbacGroup.find({ _id: { $in: groupIds }, orgId: oid, status: 'active', isGlobal: false })
+    .select('_id')
+    .lean();
+
+  return groups.map((g) => String(g._id));
+}
+
+function pickMax(values) {
+  let best = null;
+  for (const v of values) {
+    const n = toPositiveIntOrNull(v);
+    if (n === null) continue;
+    if (best === null || n > best) best = n;
+  }
+  return best;
+}
+
+function getOrgConfig(policy, orgId) {
+  const orgs = policy?.orgs || {};
+  const orgConfig = orgs[String(orgId)] || null;
+  return orgConfig && typeof orgConfig === 'object' ? orgConfig : null;
+}
+
+function getGroupConfig(orgConfig, groupId) {
+  const groups = orgConfig?.groups || {};
+  const g = groups[String(groupId)] || null;
+  return g && typeof g === 'object' ? g : null;
+}
+
+function getUserConfig(orgConfig, userId) {
+  const users = orgConfig?.users || {};
+  const u = users[String(userId)] || null;
+  return u && typeof u === 'object' ? u : null;
+}
+
+async function resolveEffectiveLimits({ userId, orgId, driveType, driveId }) {
+  const dt = normalizeDriveType(driveType);
+  const oid = normalizeObjectId(orgId, 'orgId');
+  const did = normalizeObjectId(driveId, 'driveId');
+
+  const policy = await loadPolicy();
+  const orgConfig = getOrgConfig(policy, String(oid));
+
+  const source = { maxUpload: 'default', maxStorage: 'default' };
+
+  const defaultMaxUpload = await getDefaultMaxUploadBytes();
+  const defaultMaxStorage = getDefaultMaxStorageBytes();
+
+  const globalMaxUpload = toPositiveIntOrNull(policy?.global?.maxUploadBytes);
+  const globalMaxStorage = toPositiveIntOrNull(policy?.global?.maxStorageBytes);
+
+  let maxUploadBytes = null;
+  let maxStorageBytes = null;
+
+  const orgMaxUpload = toPositiveIntOrNull(orgConfig?.maxUploadBytes);
+  const orgMaxStorage = toPositiveIntOrNull(orgConfig?.maxStorageBytes);
+
+  if (dt === 'org') {
+    maxUploadBytes = orgMaxUpload;
+    if (maxUploadBytes !== null) source.maxUpload = 'org';
+
+    maxStorageBytes = orgMaxStorage;
+    if (maxStorageBytes !== null) source.maxStorage = 'org';
+  }
+
+  if (dt === 'group') {
+    const groupConfig = getGroupConfig(orgConfig, String(did));
+    const groupMaxUpload = toPositiveIntOrNull(groupConfig?.maxUploadBytes);
+    const groupMaxStorage = toPositiveIntOrNull(groupConfig?.maxStorageBytes);
+
+    maxUploadBytes = groupMaxUpload;
+    if (maxUploadBytes !== null) source.maxUpload = 'group';
+
+    maxStorageBytes = groupMaxStorage;
+    if (maxStorageBytes !== null) source.maxStorage = 'group';
+
+    if (maxUploadBytes === null) {
+      maxUploadBytes = orgMaxUpload;
+      if (maxUploadBytes !== null) source.maxUpload = 'org';
+    }
+
+    if (maxStorageBytes === null) {
+      maxStorageBytes = orgMaxStorage;
+      if (maxStorageBytes !== null) source.maxStorage = 'org';
+    }
+  }
+
+  if (dt === 'user') {
+    const userConfig = getUserConfig(orgConfig, String(did));
+    const userMaxUpload = toPositiveIntOrNull(userConfig?.maxUploadBytes);
+    const userMaxStorage = toPositiveIntOrNull(userConfig?.maxStorageBytes);
+
+    maxUploadBytes = userMaxUpload;
+    if (maxUploadBytes !== null) source.maxUpload = 'user';
+
+    maxStorageBytes = userMaxStorage;
+    if (maxStorageBytes !== null) source.maxStorage = 'user';
+
+    const groupIds = await getUserOrgGroupIds({ userId, orgId: oid });
+    if (groupIds.length) {
+      const groupUploads = groupIds.map((gid) => getGroupConfig(orgConfig, gid)?.maxUploadBytes);
+      const groupStorages = groupIds.map((gid) => getGroupConfig(orgConfig, gid)?.maxStorageBytes);
+
+      if (maxUploadBytes === null) {
+        maxUploadBytes = pickMax(groupUploads);
+        if (maxUploadBytes !== null) source.maxUpload = 'group';
+      }
+
+      if (maxStorageBytes === null) {
+        maxStorageBytes = pickMax(groupStorages);
+        if (maxStorageBytes !== null) source.maxStorage = 'group';
+      }
+    }
+
+    if (maxUploadBytes === null) {
+      maxUploadBytes = orgMaxUpload;
+      if (maxUploadBytes !== null) source.maxUpload = 'org';
+    }
+
+    if (maxStorageBytes === null) {
+      maxStorageBytes = orgMaxStorage;
+      if (maxStorageBytes !== null) source.maxStorage = 'org';
+    }
+  }
+
+  if (maxUploadBytes === null && globalMaxUpload !== null) {
+    maxUploadBytes = globalMaxUpload;
+    source.maxUpload = 'global';
+  }
+
+  if (maxStorageBytes === null && globalMaxStorage !== null) {
+    maxStorageBytes = globalMaxStorage;
+    source.maxStorage = 'global';
+  }
+
+  if (maxUploadBytes === null) {
+    maxUploadBytes = defaultMaxUpload;
+  }
+
+  if (maxStorageBytes === null) {
+    maxStorageBytes = defaultMaxStorage;
+  }
+
+  return {
+    maxUploadBytes,
+    maxStorageBytes,
+    source,
+  };
+}
+
+async function computeDriveUsedBytes({ orgId, driveType, driveId }) {
+  const oid = normalizeObjectId(orgId, 'orgId');
+  const dt = normalizeDriveType(driveType);
+  const did = normalizeObjectId(driveId, 'driveId');
+
+  const rows = await FileEntry.aggregate([
+    {
+      $match: {
+        orgId: oid,
+        driveType: dt,
+        driveId: did,
+        deletedAt: null,
+      },
+    },
+    {
+      $lookup: {
+        from: 'assets',
+        localField: 'assetId',
+        foreignField: '_id',
+        as: 'asset',
+      },
+    },
+    { $unwind: { path: '$asset', preserveNullAndEmptyArrays: false } },
+    { $match: { 'asset.status': 'uploaded' } },
+    {
+      $group: {
+        _id: null,
+        usedBytes: { $sum: '$asset.sizeBytes' },
+      },
+    },
+  ]);
+
+  const usedBytes = rows?.[0]?.usedBytes;
+  return Number.isFinite(usedBytes) ? usedBytes : 0;
+}
+
+async function getEffectivePolicy({ userId, orgId, driveType, driveId }) {
+  const effective = await resolveEffectiveLimits({ userId, orgId, driveType, driveId });
+  const usedBytes = await computeDriveUsedBytes({ orgId, driveType, driveId });
+  const overageBytes = Math.max(0, usedBytes - effective.maxStorageBytes);
+
+  return {
+    effective,
+    usage: { usedBytes, overageBytes },
+  };
+}
+
+module.exports = {
+  loadPolicy,
+  resolveEffectiveLimits,
+  computeDriveUsedBytes,
+  getEffectivePolicy,
+  getUserOrgGroupIds,
+};

package/src/services/headlessExternalModels.service.js

@@ -0,0 +1,292 @@
+const mongoose = require('mongoose');
+
+const HeadlessModelDefinition = require('../models/HeadlessModelDefinition');
+const { normalizeCodeIdentifier, computeSchemaHash } = require('./headlessModels.service');
+
+function isObjectId(value) {
+  if (!value) return false;
+  if (value instanceof mongoose.Types.ObjectId) return true;
+  if (value && typeof value === 'object' && value._bsontype === 'ObjectID') return true;
+  return false;
+}
+
+function detectFieldType(value) {
+  if (value === null || value === undefined) return null;
+  if (isObjectId(value)) return 'objectid';
+  if (value instanceof Date) return 'date';
+  if (Array.isArray(value)) return 'array';
+  const t = typeof value;
+  if (t === 'string') return 'string';
+  if (t === 'number') return 'number';
+  if (t === 'boolean') return 'boolean';
+  if (t === 'object') return 'object';
+  return null;
+}
+
+function pickBestType(seenTypes) {
+  const types = new Set(Array.from(seenTypes || []).filter(Boolean));
+  if (types.size === 0) return { type: 'object', warning: 'No non-null sample values' };
+  if (types.size === 1) return { type: Array.from(types)[0], warning: null };
+  if (types.has('array')) return { type: 'array', warning: `Mixed types: ${Array.from(types).join('|')}` };
+  if (types.has('object')) return { type: 'object', warning: `Mixed types: ${Array.from(types).join('|')}` };
+  return { type: 'string', warning: `Mixed types: ${Array.from(types).join('|')}` };
+}
+
+function tryInferRefFromFieldName(fieldName, externalModelsByCollection) {
+  const name = String(fieldName || '').trim();
+  if (!name) return null;
+
+  const lower = name.toLowerCase();
+  if (!lower.endsWith('id') || lower === 'id') return null;
+
+  const stem = name.slice(0, -2);
+  const candidates = [stem, `${stem}s`, `${stem}es`];
+  for (const c of candidates) {
+    const match = externalModelsByCollection.get(String(c).toLowerCase());
+    if (match) return match;
+  }
+  return null;
+}
+
+function normalizeIndexFromMongo(idx) {
+  if (!idx || typeof idx !== 'object') return null;
+  const fields = idx.key;
+  if (!fields || typeof fields !== 'object') return null;
+
+  const options = { ...idx };
+  delete options.key;
+  delete options.v;
+  delete options.ns;
+
+  return { fields, options };
+}
+
+async function listExternalCollections({ q, includeSystem } = {}) {
+  if (!mongoose.connection || !mongoose.connection.db) {
+    const err = new Error('Mongo connection not ready');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const filter = {};
+  if (q) filter.name = { $regex: String(q), $options: 'i' };
+
+  const cursor = await mongoose.connection.db.listCollections(filter, { nameOnly: true });
+  const items = await cursor.toArray();
+
+  const out = [];
+  for (const c of items) {
+    const name = String(c && c.name ? c.name : '').trim();
+    if (!name) continue;
+    if (!includeSystem && name.startsWith('system.')) continue;
+    out.push({ name, type: 'collection' });
+  }
+
+  out.sort((a, b) => a.name.localeCompare(b.name));
+  return out;
+}
+
+async function inferExternalModelFromCollection({ collectionName, sampleSize = 200 } = {}) {
+  if (!mongoose.connection || !mongoose.connection.db) {
+    const err = new Error('Mongo connection not ready');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const collName = String(collectionName || '').trim();
+  if (!collName) {
+    const err = new Error('collectionName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const N = Math.max(1, Math.min(Number(sampleSize) || 200, 1000));
+  const coll = mongoose.connection.db.collection(collName);
+
+  let docs;
+  try {
+    docs = await coll.aggregate([{ $sample: { size: N } }]).toArray();
+  } catch {
+    docs = await coll.find({}).limit(N).toArray();
+  }
+
+  const externalModels = await HeadlessModelDefinition.find({ isActive: true, sourceType: 'external' })
+    .select({ codeIdentifier: 1, sourceCollectionName: 1 })
+    .lean();
+
+  const externalModelsByCollection = new Map();
+  for (const m of externalModels || []) {
+    const cn = String(m?.sourceCollectionName || '').trim();
+    const code = String(m?.codeIdentifier || '').trim();
+    if (cn && code) externalModelsByCollection.set(cn.toLowerCase(), code);
+  }
+
+  const perField = new Map();
+  const warnings = [];
+
+  for (const doc of docs || []) {
+    if (!doc || typeof doc !== 'object') continue;
+    for (const [k, v] of Object.entries(doc)) {
+      if (!k || k === '_id') continue;
+      const type = detectFieldType(v);
+      let stats = perField.get(k);
+      if (!stats) {
+        stats = { seenTypes: new Set(), objectIdCount: 0, nonNullCount: 0 };
+        perField.set(k, stats);
+      }
+      if (type) stats.seenTypes.add(type);
+      if (type === 'objectid') stats.objectIdCount += 1;
+      if (v !== null && v !== undefined) stats.nonNullCount += 1;
+    }
+  }
+
+  const fields = [];
+  for (const [name, stats] of perField.entries()) {
+    const { type, warning } = pickBestType(stats.seenTypes);
+
+    if (warning) warnings.push(`Field ${name}: ${warning}`);
+
+    if (type === 'objectid') {
+      const refModelCode = tryInferRefFromFieldName(name, externalModelsByCollection);
+      if (refModelCode) {
+        fields.push({ name, type: 'ref', required: false, unique: false, refModelCode });
+      } else {
+        fields.push({ name, type: 'string', required: false, unique: false });
+      }
+      continue;
+    }
+
+    if (type === 'date') {
+      fields.push({ name, type: 'date', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'array') {
+      fields.push({ name, type: 'array', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'object') {
+      fields.push({ name, type: 'object', required: false, unique: false });
+      continue;
+    }
+
+    if (type === 'number' || type === 'boolean' || type === 'string') {
+      fields.push({ name, type, required: false, unique: false });
+      continue;
+    }
+
+    fields.push({ name, type: 'object', required: false, unique: false });
+  }
+
+  fields.sort((a, b) => a.name.localeCompare(b.name));
+
+  let indexes = [];
+  try {
+    const idx = await coll.indexes();
+    indexes = (idx || []).map(normalizeIndexFromMongo).filter(Boolean);
+  } catch {
+    indexes = [];
+  }
+
+  const fieldsHash = computeSchemaHash({ fields, indexes });
+
+  return {
+    collectionName: collName,
+    fields,
+    indexes,
+    warnings,
+    stats: {
+      sampled: (docs || []).length,
+      maxSampleSize: N,
+      fields: fields.length,
+    },
+    fieldsHash,
+  };
+}
+
+async function createOrUpdateExternalModel({ collectionName, codeIdentifier, displayName, sampleSize } = {}) {
+  const cn = String(collectionName || '').trim();
+  if (!cn) {
+    const err = new Error('collectionName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const code = normalizeCodeIdentifier(codeIdentifier);
+  if (!code.startsWith('ext_')) {
+    const err = new Error('External model codeIdentifier must start with ext_');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const name = String(displayName || code).trim();
+  if (!name) {
+    const err = new Error('displayName is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const inferred = await inferExternalModelFromCollection({ collectionName: cn, sampleSize });
+
+  const existing = await HeadlessModelDefinition.findOne({ codeIdentifier: code, isActive: true });
+  if (!existing) {
+    const doc = await HeadlessModelDefinition.create({
+      codeIdentifier: code,
+      displayName: name,
+      description: '',
+      fields: inferred.fields,
+      indexes: inferred.indexes,
+      fieldsHash: inferred.fieldsHash,
+      version: 1,
+      previousFields: [],
+      previousIndexes: [],
+      sourceType: 'external',
+      sourceCollectionName: cn,
+      isExternal: true,
+      inference: {
+        enabled: true,
+        lastInferredAt: new Date(),
+        sampleSize: Number(sampleSize) || null,
+        warnings: inferred.warnings || [],
+        stats: inferred.stats || null,
+      },
+      isActive: true,
+    });
+
+    return { created: true, item: doc.toObject(), inference: inferred };
+  }
+
+  existing.displayName = name;
+  existing.sourceType = 'external';
+  existing.sourceCollectionName = cn;
+  existing.isExternal = true;
+
+  const newHash = inferred.fieldsHash;
+  if (newHash !== existing.fieldsHash) {
+    existing.previousFields = existing.fields;
+    existing.previousIndexes = existing.indexes;
+    existing.fields = inferred.fields;
+    existing.indexes = inferred.indexes;
+    existing.fieldsHash = newHash;
+    existing.version = Number(existing.version || 1) + 1;
+  }
+
+  existing.inference = {
+    enabled: true,
+    lastInferredAt: new Date(),
+    sampleSize: Number(sampleSize) || null,
+    warnings: inferred.warnings || [],
+    stats: inferred.stats || null,
+  };
+
+  await existing.save();
+
+  return { created: false, item: existing.toObject(), inference: inferred };
+}
+
+module.exports = {
+  listExternalCollections,
+  inferExternalModelFromCollection,
+  createOrUpdateExternalModel,
+};

package/src/services/headlessModels.service.js

@@ -89,6 +89,18 @@ function getMongoCollectionName(codeIdentifier) {
   return `${MODEL_COLLECTION_PREFIX}${code}`;
 }
 
+function isExternalDefinition(def) {
+  return def && (def.sourceType === 'external' || def.isExternal === true);
+}
+
+function getCollectionNameForDefinition(def) {
+  if (isExternalDefinition(def)) {
+    const cn = String(def.sourceCollectionName || '').trim();
+    return cn || getMongoCollectionName(def.codeIdentifier);
+  }
+  return getMongoCollectionName(def.codeIdentifier);
+}
+
 function buildSchemaFromDefinition(def) {
   const schemaShape = {};
   for (const field of def.fields || []) {
@@ -99,12 +111,14 @@ function buildSchemaFromDefinition(def) {
     schemaShape[fieldName] = toMongooseField(field);
   }
 
-  schemaShape._headlessModelCode = { type: String, default: def.codeIdentifier, index: true };
-  schemaShape._headlessSchemaVersion = { type: Number, default: def.version, index: true };
+  if (!isExternalDefinition(def)) {
+    schemaShape._headlessModelCode = { type: String, default: def.codeIdentifier, index: true };
+    schemaShape._headlessSchemaVersion = { type: Number, default: def.version, index: true };
+  }
 
   const schema = new mongoose.Schema(schemaShape, {
     timestamps: true,
-    collection: getMongoCollectionName(def.codeIdentifier),
+    collection: getCollectionNameForDefinition(def),
     strict: false,
   });
 
@@ -254,6 +268,8 @@ async function disableModelDefinition(codeIdentifier) {
 const modelCache = new Map();
 
 async function ensureAutoMigration(modelDef) {
+  if (isExternalDefinition(modelDef)) return;
+
   const collectionName = getMongoCollectionName(modelDef.codeIdentifier);
   const coll = mongoose.connection.collection(collectionName);
 
@@ -318,7 +334,8 @@ async function getDynamicModel(codeIdentifier) {
     throw err;
   }
 
-  const cacheKey = `${def.codeIdentifier}:${def.version}:${def.fieldsHash}`;
+  const collectionName = getCollectionNameForDefinition(def);
+  const cacheKey = `${def.codeIdentifier}:${def.version}:${def.fieldsHash}:${collectionName}`;
   const cached = modelCache.get(cacheKey);
   if (cached) return cached;
 
@@ -334,8 +351,10 @@ async function getDynamicModel(codeIdentifier) {
   const schema = buildSchemaFromDefinition(def);
   const Model = mongoose.model(modelName, schema);
 
-  await ensureAutoMigration(def);
-  await ensureIndexesBestEffort(Model);
+  if (!isExternalDefinition(def)) {
+    await ensureAutoMigration(def);
+    await ensureIndexesBestEffort(Model);
+  }
 
   modelCache.set(cacheKey, Model);
   return Model;
@@ -346,6 +365,7 @@ module.exports = {
   normalizeCodeIdentifier,
   getMongooseModelName,
   getMongoCollectionName,
+  getCollectionNameForDefinition,
   computeSchemaHash,
   listModelDefinitions,
   getModelDefinitionByCode,