@intranefr/superbackend 1.5.0 → 1.5.2

package/src/services/ejsVirtual.service.js

@@ -8,7 +8,7 @@ const VirtualEjsFileVersion = require('../models/VirtualEjsFileVersion');
 const VirtualEjsGroupChange = require('../models/VirtualEjsGroupChange');
 
 const llmService = require('./llm.service');
-const { getSettingValue } = require('./globalSettings.service');
+const { resolveLlmProviderModel } = require('./llmDefaults.service');
 const { createAuditEvent } = require('./audit.service');
 
 const CACHE_TTL_MS = 5 * 60 * 1000;
@@ -151,6 +151,7 @@ async function resolveTemplateSource({ viewsRoot, relPath, allowDb = true }) {
   }
 
   if (fileDoc && fileDoc.enabled === true && typeof fileDoc.content === 'string' && fileDoc.content.trim() !== '') {
+    console.log(`[ejsVirtual] Resolved ${normalized} from DB (${fileDoc.content.length} chars)`);
     return {
       relPath: normalized,
       source: 'db',
@@ -160,6 +161,7 @@ async function resolveTemplateSource({ viewsRoot, relPath, allowDb = true }) {
   }
 
   const fsContent = await readFsView(viewsRoot, normalized);
+  console.log(`[ejsVirtual] Resolved ${normalized} from FS (${fsContent?.length || 0} chars)`);
   return {
     relPath: normalized,
     source: 'fs',
@@ -259,25 +261,11 @@ function parseMultiFilePatch(text) {
 }
 
 async function resolveLlmDefaults({ providerKey, model }) {
-  const uiProvider = String(providerKey || '').trim();
-  const uiModel = String(model || '').trim();
-
-  const settingProvider = String(await getSettingValue('ejsVirtual.ai.providerKey', '') || '').trim();
-  const settingModel = String(await getSettingValue('ejsVirtual.ai.model', '') || '').trim();
-
-  const envProvider = String(process.env.DEFAULT_LLM_PROVIDER_KEY || '').trim();
-  const envModel = String(process.env.DEFAULT_LLM_MODEL || '').trim();
-
-  const resolvedProviderKey = uiProvider || settingProvider || envProvider;
-  if (!resolvedProviderKey) {
-    const err = new Error('Missing LLM providerKey (configure ejsVirtual.ai.providerKey or DEFAULT_LLM_PROVIDER_KEY, or send from UI)');
-    err.code = 'VALIDATION';
-    throw err;
-  }
-
-  const resolvedModel = uiModel || settingModel || envModel || 'x-ai/grok-code-fast-1';
-
-  return { providerKey: resolvedProviderKey, model: resolvedModel };
+  return resolveLlmProviderModel({
+    systemKey: 'ejsVirtual.vibe.apply',
+    providerKey,
+    model,
+  });
 }
 
 async function vibeEdit({ prompt, paths, providerKey, model, viewsRoot, actor }) {
@@ -492,8 +480,9 @@ function resolveIncludeRelPath({ viewsRoot, parentAbsPath, includePath }) {
   }
 
   const normalizedAbs = path.resolve(includeAbs);
-  if (!normalizedAbs.startsWith(baseRoot)) {
-    const err = new Error('Include path escapes views root');
+  // Use path.sep to ensure we match the full directory name
+  if (!normalizedAbs.startsWith(baseRoot + path.sep) && normalizedAbs !== baseRoot) {
+    const err = new Error(`Include path escapes views root: ${normalizedAbs} (root: ${baseRoot})`);
     err.code = 'VALIDATION';
     throw err;
   }
@@ -505,24 +494,62 @@ function resolveIncludeRelPath({ viewsRoot, parentAbsPath, includePath }) {
 async function preloadTemplatesForRender({ viewsRoot, entryRelPath }) {
   const templatesByRelPath = new Map();
   const absByRelPath = new Map();
-  const queue = [entryRelPath];
   const seen = new Set();
 
+  // 1. Pre-populate with all enabled DB overrides.
+  // This ensures dynamic includes (like blocks) that exist in DB are available.
+  try {
+    const dbFiles = await VirtualEjsFile.find({ enabled: true }).lean();
+    if (dbFiles && dbFiles.length > 0) {
+      console.log(`[ejsVirtual] Preloading ${dbFiles.length} enabled DB templates: ${dbFiles.map(f => `${f.path} (${f.content?.length || 0} chars)`).join(', ')}`);
+      for (const f of dbFiles) {
+        templatesByRelPath.set(f.path, {
+          relPath: f.path,
+          source: 'db',
+          content: f.content,
+          updatedAt: f.updatedAt,
+        });
+        absByRelPath.set(f.path, resolveAbsPath(viewsRoot, f.path));
+      }
+    }
+  } catch (err) {
+    console.error('[ejsVirtual] Failed to preload DB templates:', err);
+  }
+
+  const queue = [entryRelPath];
+  // Also crawl static includes of DB-overridden files
+  for (const p of templatesByRelPath.keys()) {
+    queue.push(p);
+  }
+
   while (queue.length > 0) {
     const relPath = queue.shift();
     if (!relPath || seen.has(relPath)) continue;
     seen.add(relPath);
 
-    const src = await resolveTemplateSource({ viewsRoot, relPath, allowDb: true });
-    const abs = resolveAbsPath(viewsRoot, relPath);
+    let src = templatesByRelPath.get(relPath);
+    if (!src) {
+      try {
+        src = await resolveTemplateSource({ viewsRoot, relPath, allowDb: true });
+        templatesByRelPath.set(relPath, src);
+      } catch (err) {
+        // If not found, we still want to keep going, it might be on FS or missing
+        console.warn(`[ejsVirtual] Failed to resolve template source for ${relPath}:`, err.message);
+        continue;
+      }
+    }
 
-    templatesByRelPath.set(relPath, src);
+    const abs = absByRelPath.get(relPath) || resolveAbsPath(viewsRoot, relPath);
     absByRelPath.set(relPath, abs);
 
     const includes = extractIncludePaths(src.content);
     for (const inc of includes) {
-      const incRel = resolveIncludeRelPath({ viewsRoot, parentAbsPath: abs, includePath: inc });
-      if (incRel) queue.push(incRel);
+      try {
+        const incRel = resolveIncludeRelPath({ viewsRoot, parentAbsPath: abs, includePath: inc });
+        if (incRel) queue.push(incRel);
+      } catch (err) {
+        console.warn(`[ejsVirtual] Failed to resolve include path "${inc}" in ${relPath}:`, err.message);
+      }
     }
   }
 
@@ -533,18 +560,26 @@ async function renderToString(res, viewPath, data = {}, options = {}) {
   const relPath = normalizeViewPath(viewPath);
   const viewsRoot = path.resolve(options.viewsRoot || (res && res.app ? res.app.get('views') : null) || getDefaultViewsRoot());
 
+  console.log(`[ejsVirtual] Rendering ${relPath} (viewsRoot: ${viewsRoot})`);
+
   const { templatesByRelPath, absByRelPath } = await preloadTemplatesForRender({
     viewsRoot,
     entryRelPath: relPath,
   });
 
+  if (templatesByRelPath.size === 0) {
+    console.warn(`[ejsVirtual] No templates preloaded for ${relPath}`);
+  }
+
   const entry = templatesByRelPath.get(relPath);
   if (!entry) {
-    const err = new Error('Template not found');
+    const err = new Error(`Template not found: ${relPath}`);
     err.code = 'NOT_FOUND';
     throw err;
   }
 
+  console.log(`[ejsVirtual] Entry template ${relPath} source: ${entry.source}`);
+
   const entryAbs = absByRelPath.get(relPath) || resolveAbsPath(viewsRoot, relPath);
 
   const versionKey = entry.source === 'db'
@@ -553,6 +588,13 @@ async function renderToString(res, viewPath, data = {}, options = {}) {
 
   const cached = getCachedTemplate(entry.relPath, versionKey);
   const entryTemplate = cached ? cached.template : entry.content;
+  
+  if (!entryTemplate || entryTemplate.trim() === '') {
+    console.warn(`[ejsVirtual] Entry template ${relPath} is empty! Source: ${entry.source}`);
+  } else {
+    console.log(`[ejsVirtual] Entry template ${relPath} content start: ${entryTemplate.substring(0, 50).replace(/\n/g, '\\n')}...`);
+  }
+
   if (!cached) {
     setCachedTemplate(entry.relPath, versionKey, entry.content, {});
   }
@@ -577,10 +619,29 @@ async function renderToString(res, viewPath, data = {}, options = {}) {
 
     const incAbs = incRel ? absByRelPath.get(incRel) || resolveAbsPath(viewsRoot, incRel) : parentAbs;
     const src = incRel ? templatesByRelPath.get(incRel) : null;
+    
     if (!incRel || !src) {
-      return { filename: incAbs };
+      // If not in preloaded map, it might be on FS.
+      // Check if it exists on FS to avoid ENOENT crash
+      try {
+        if (incAbs && fs.existsSync(incAbs) && fs.statSync(incAbs).isFile()) {
+          console.log(`[ejsVirtual] Include ${incRel || includePath} found on FS (fallback)`);
+          return { filename: incAbs };
+        }
+      } catch (e) {
+        // ignore errors
+      }
+      
+      // If it doesn't exist anywhere, return a comment instead of crashing
+      console.warn(`[ejsVirtual] Include not found: ${incRel || includePath} (parent: ${parentAbs})`);
+      return { template: `<!-- Include not found: ${incRel || includePath} -->` };
+    }
+
+    if (src.content === undefined || src.content === null || src.content.trim() === '') {
+      console.warn(`[ejsVirtual] Including ${incRel} from ${src.source} but content is empty`);
     }
 
+    console.log(`[ejsVirtual] Including ${incRel} from ${src.source}`);
     return {
       filename: incAbs,
       template: src.content,
@@ -589,11 +650,19 @@ async function renderToString(res, viewPath, data = {}, options = {}) {
 
   await recordIntegratedUsage(relPath, null);
 
-  return ejs.render(entryTemplate, { ...(data || {}), ...(res?.locals || {}) }, {
+  const rendered = ejs.render(entryTemplate, { ...(data || {}), ...(res?.locals || {}) }, {
     filename: entryAbs,
     async: false,
     includer,
   });
+
+  if (!rendered || rendered.trim() === '') {
+    console.warn(`[ejsVirtual] Rendered content for ${relPath} is empty!`);
+  } else {
+    console.log(`[ejsVirtual] Rendered content for ${relPath} length: ${rendered.length} chars`);
+  }
+
+  return rendered;
 }
 
 async function render(res, viewPath, data = {}, options = {}) {
@@ -608,7 +677,8 @@ module.exports = {
   readFsView,
   invalidateCacheForPath,
   clearCache,
+  recordIntegratedUsage,
   vibeEdit,
   renderToString,
   render,
-};
+};

package/src/services/experiments.service.js

@@ -0,0 +1,273 @@
+const crypto = require('crypto');
+const mongoose = require('mongoose');
+
+const Experiment = require('../models/Experiment');
+const ExperimentAssignment = require('../models/ExperimentAssignment');
+const ExperimentEvent = require('../models/ExperimentEvent');
+
+const cacheLayer = require('./cacheLayer.service');
+const jsonConfigsService = require('./jsonConfigs.service');
+
+function normalizeStr(v) {
+  return String(v || '').trim();
+}
+
+function normalizeOrgId(orgId) {
+  if (orgId === null || orgId === undefined || orgId === '') return null;
+  const str = String(orgId);
+  if (!mongoose.Types.ObjectId.isValid(str)) {
+    const err = new Error('Invalid orgId');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+  return new mongoose.Types.ObjectId(str);
+}
+
+function normalizeExperimentCode(code) {
+  const c = normalizeStr(code);
+  if (!c) {
+    const err = new Error('experiment code is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+  return c;
+}
+
+function normalizeSubjectId(subjectId) {
+  const s = normalizeStr(subjectId);
+  if (!s) {
+    const err = new Error('subjectId is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+  return s;
+}
+
+function computeSubjectKey({ orgId, subjectId }) {
+  const sid = normalizeSubjectId(subjectId);
+  const oid = orgId ? String(orgId) : 'global';
+  return `org:${oid}:subject:${sid}`;
+}
+
+function computeBucketInt(input, max) {
+  const hash = crypto.createHash('sha256').update(String(input), 'utf8').digest('hex');
+  const int = parseInt(hash.slice(0, 8), 16);
+  return max <= 0 ? 0 : int % max;
+}
+
+function pickWeightedVariant({ experiment, subjectKey }) {
+  const variants = Array.isArray(experiment?.variants) ? experiment.variants : [];
+  const eligible = variants
+    .map((v) => ({
+      key: normalizeStr(v?.key),
+      weight: Number(v?.weight || 0) || 0,
+      configSlug: normalizeStr(v?.configSlug),
+    }))
+    .filter((v) => v.key && v.weight > 0);
+
+  if (!eligible.length) {
+    const err = new Error('Experiment has no weighted variants');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const total = eligible.reduce((acc, v) => acc + v.weight, 0);
+  const salt = normalizeStr(experiment?.assignment?.salt) || String(experiment?._id || '');
+  const pos = computeBucketInt(`${salt}:${subjectKey}`, total);
+
+  let cursor = 0;
+  for (const v of eligible) {
+    cursor += v.weight;
+    if (pos < cursor) return v;
+  }
+
+  return eligible[eligible.length - 1];
+}
+
+async function getExperimentByCode({ orgId, code }) {
+  const c = normalizeExperimentCode(code);
+  const oid = orgId ? normalizeOrgId(orgId) : null;
+
+  const doc = await Experiment.findOne({ organizationId: oid, code: c }).lean();
+  if (doc) return doc;
+
+  if (oid) {
+    const globalDoc = await Experiment.findOne({ organizationId: null, code: c }).lean();
+    if (globalDoc) return globalDoc;
+  }
+
+  const err = new Error('Experiment not found');
+  err.code = 'NOT_FOUND';
+  throw err;
+}
+
+async function resolveVariantConfig(variant) {
+  const slug = normalizeStr(variant?.configSlug);
+  if (!slug) return null;
+  return jsonConfigsService.getJsonConfigValueBySlug(slug);
+}
+
+async function getOrCreateAssignment({ orgId, experimentCode, subjectId, context }) {
+  const exp = await getExperimentByCode({ orgId, code: experimentCode });
+  const effectiveOrgId = exp.organizationId ? String(exp.organizationId) : null;
+  const subjectKey = computeSubjectKey({ orgId: effectiveOrgId || orgId || null, subjectId });
+
+  const cacheKey = `${String(exp._id)}:${subjectKey}`;
+  const cached = await cacheLayer.get(cacheKey, { namespace: 'experiments.assignments' });
+  if (cached && cached.variantKey) return { experiment: exp, assignment: cached };
+
+  const existing = await ExperimentAssignment.findOne({ experimentId: exp._id, subjectKey }).lean();
+  if (existing) {
+    const assignment = {
+      experimentId: String(existing.experimentId),
+      organizationId: existing.organizationId ? String(existing.organizationId) : null,
+      subjectKey: existing.subjectKey,
+      variantKey: existing.variantKey,
+      assignedAt: existing.assignedAt,
+      context: existing.context || {},
+    };
+    await cacheLayer.set(cacheKey, assignment, { namespace: 'experiments.assignments', ttlSeconds: 60 });
+    return { experiment: exp, assignment };
+  }
+
+  if (exp.status !== 'running' && exp.status !== 'completed') {
+    const err = new Error('Experiment is not active');
+    err.code = 'CONFLICT';
+    throw err;
+  }
+
+  const picked = pickWeightedVariant({ experiment: exp, subjectKey });
+
+  const created = await ExperimentAssignment.create({
+    experimentId: exp._id,
+    organizationId: exp.organizationId || null,
+    subjectKey,
+    variantKey: picked.key,
+    assignedAt: new Date(),
+    context: context && typeof context === 'object' ? context : {},
+  });
+
+  const assignment = {
+    experimentId: String(created.experimentId),
+    organizationId: created.organizationId ? String(created.organizationId) : null,
+    subjectKey: created.subjectKey,
+    variantKey: created.variantKey,
+    assignedAt: created.assignedAt,
+    context: created.context || {},
+  };
+
+  await cacheLayer.set(cacheKey, assignment, { namespace: 'experiments.assignments', ttlSeconds: 60 });
+  return { experiment: exp, assignment };
+}
+
+async function ingestEvents({ orgId, experimentCode, subjectId, events }) {
+  const exp = await getExperimentByCode({ orgId, code: experimentCode });
+
+  const effectiveOrgId = exp.organizationId ? String(exp.organizationId) : null;
+  const subjectKey = computeSubjectKey({ orgId: effectiveOrgId || orgId || null, subjectId });
+
+  const list = Array.isArray(events) ? events : [];
+  if (!list.length) {
+    const err = new Error('events[] is required');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const variantKeys = new Set((exp.variants || []).map((v) => String(v?.key || '').trim()).filter(Boolean));
+
+  const now = new Date();
+  const docs = [];
+  for (const e of list) {
+    if (!e || typeof e !== 'object') continue;
+
+    const eventKey = normalizeStr(e.eventKey);
+    if (!eventKey) {
+      const err = new Error('eventKey is required');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+
+    const ts = e.ts ? new Date(e.ts) : now;
+    if (!Number.isFinite(ts.getTime())) {
+      const err = new Error('Invalid ts');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+
+    let variantKey = normalizeStr(e.variantKey);
+    if (!variantKey) {
+      const { assignment } = await getOrCreateAssignment({ orgId, experimentCode, subjectId, context: null });
+      variantKey = assignment.variantKey;
+    }
+
+    if (!variantKeys.has(variantKey)) {
+      const err = new Error('Invalid variantKey');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+
+    const value = e.value === undefined ? 1 : Number(e.value);
+    if (!Number.isFinite(value)) {
+      const err = new Error('Invalid value');
+      err.code = 'VALIDATION';
+      throw err;
+    }
+
+    docs.push({
+      experimentId: exp._id,
+      organizationId: exp.organizationId || null,
+      subjectKey,
+      variantKey,
+      eventKey,
+      value,
+      ts,
+      meta: e.meta && typeof e.meta === 'object' ? e.meta : {},
+    });
+  }
+
+  if (!docs.length) {
+    const err = new Error('No valid events provided');
+    err.code = 'VALIDATION';
+    throw err;
+  }
+
+  const inserted = await ExperimentEvent.insertMany(docs, { ordered: false });
+  return { insertedCount: Array.isArray(inserted) ? inserted.length : 0 };
+}
+
+async function getWinnerSnapshot({ orgId, experimentCode }) {
+  const exp = await getExperimentByCode({ orgId, code: experimentCode });
+
+  const cacheKey = String(exp._id);
+  const cached = await cacheLayer.get(cacheKey, { namespace: 'experiments.winner' });
+  if (cached && typeof cached === 'object') return { experiment: exp, snapshot: cached };
+
+  const snapshot = {
+    experimentId: String(exp._id),
+    organizationId: exp.organizationId ? String(exp.organizationId) : null,
+    code: exp.code,
+    status: exp.status,
+    winnerVariantKey: exp.winnerVariantKey || null,
+    winnerDecidedAt: exp.winnerDecidedAt || null,
+    winnerReason: exp.winnerReason || null,
+  };
+
+  await cacheLayer.set(cacheKey, snapshot, { namespace: 'experiments.winner', ttlSeconds: 30 });
+  return { experiment: exp, snapshot };
+}
+
+async function clearExperimentCaches(experimentId) {
+  const id = String(experimentId || '').trim();
+  if (!id) return;
+  await cacheLayer.delete(id, { namespace: 'experiments.winner' }).catch(() => {});
+}
+
+module.exports = {
+  computeSubjectKey,
+  getExperimentByCode,
+  resolveVariantConfig,
+  getOrCreateAssignment,
+  ingestEvents,
+  getWinnerSnapshot,
+  clearExperimentCaches,
+};