@intranefr/superbackend 1.5.1 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/index.js +2 -0
- package/manage.js +745 -0
- package/package.json +5 -2
- package/src/controllers/admin.controller.js +79 -6
- package/src/controllers/adminAgents.controller.js +37 -0
- package/src/controllers/adminExperiments.controller.js +200 -0
- package/src/controllers/adminLlm.controller.js +19 -0
- package/src/controllers/adminMarkdowns.controller.js +157 -0
- package/src/controllers/adminScripts.controller.js +243 -74
- package/src/controllers/adminTelegram.controller.js +72 -0
- package/src/controllers/experiments.controller.js +85 -0
- package/src/controllers/internalExperiments.controller.js +17 -0
- package/src/controllers/markdowns.controller.js +42 -0
- package/src/helpers/mongooseHelper.js +258 -0
- package/src/helpers/scriptBase.js +230 -0
- package/src/helpers/scriptRunner.js +335 -0
- package/src/middleware.js +195 -34
- package/src/models/Agent.js +105 -0
- package/src/models/AgentMessage.js +82 -0
- package/src/models/CacheEntry.js +1 -1
- package/src/models/ConsoleLog.js +1 -1
- package/src/models/Experiment.js +75 -0
- package/src/models/ExperimentAssignment.js +23 -0
- package/src/models/ExperimentEvent.js +26 -0
- package/src/models/ExperimentMetricBucket.js +30 -0
- package/src/models/GlobalSetting.js +1 -2
- package/src/models/Markdown.js +75 -0
- package/src/models/RateLimitCounter.js +1 -1
- package/src/models/ScriptDefinition.js +1 -0
- package/src/models/ScriptRun.js +8 -0
- package/src/models/TelegramBot.js +42 -0
- package/src/models/Webhook.js +2 -0
- package/src/routes/admin.routes.js +2 -0
- package/src/routes/adminAgents.routes.js +13 -0
- package/src/routes/adminConsoleManager.routes.js +1 -1
- package/src/routes/adminExperiments.routes.js +29 -0
- package/src/routes/adminLlm.routes.js +1 -0
- package/src/routes/adminMarkdowns.routes.js +16 -0
- package/src/routes/adminScripts.routes.js +4 -1
- package/src/routes/adminTelegram.routes.js +14 -0
- package/src/routes/blogInternal.routes.js +2 -2
- package/src/routes/experiments.routes.js +30 -0
- package/src/routes/internalExperiments.routes.js +15 -0
- package/src/routes/markdowns.routes.js +16 -0
- package/src/services/agent.service.js +546 -0
- package/src/services/agentHistory.service.js +345 -0
- package/src/services/agentTools.service.js +578 -0
- package/src/services/blogCronsBootstrap.service.js +7 -6
- package/src/services/consoleManager.service.js +56 -18
- package/src/services/consoleOverride.service.js +1 -0
- package/src/services/experiments.service.js +273 -0
- package/src/services/experimentsAggregation.service.js +308 -0
- package/src/services/experimentsCronsBootstrap.service.js +118 -0
- package/src/services/experimentsRetention.service.js +43 -0
- package/src/services/experimentsWs.service.js +134 -0
- package/src/services/globalSettings.service.js +15 -0
- package/src/services/jsonConfigs.service.js +24 -12
- package/src/services/llm.service.js +219 -6
- package/src/services/markdowns.service.js +522 -0
- package/src/services/scriptsRunner.service.js +514 -23
- package/src/services/telegram.service.js +130 -0
- package/src/utils/rbac/rightsRegistry.js +4 -0
- package/views/admin-agents.ejs +273 -0
- package/views/admin-coolify-deploy.ejs +8 -8
- package/views/admin-dashboard.ejs +63 -12
- package/views/admin-experiments.ejs +91 -0
- package/views/admin-markdowns.ejs +905 -0
- package/views/admin-scripts.ejs +817 -6
- package/views/admin-telegram.ejs +269 -0
- package/views/partials/dashboard/nav-items.ejs +4 -0
- package/views/partials/dashboard/palette.ejs +5 -3
- package/src/middleware/internalCronAuth.js +0 -29
|
@@ -13,6 +13,21 @@ const { logErrorSync } = require("./errorLogger");
|
|
|
13
13
|
const CronJob = require("../models/CronJob");
|
|
14
14
|
const ScriptDefinition = require("../models/ScriptDefinition");
|
|
15
15
|
|
|
16
|
+
// Import consoleOverride to access the truly original console
|
|
17
|
+
const consoleOverride = require("./consoleOverride.service");
|
|
18
|
+
|
|
19
|
+
// Simplified module prefix tracking
|
|
20
|
+
let currentModulePrefix = '';
|
|
21
|
+
|
|
22
|
+
// Module prefix mapping based on module name
|
|
23
|
+
const MODULE_PREFIXES = {
|
|
24
|
+
'cronScheduler.service.js': '[SuperBackend][Cron]',
|
|
25
|
+
'healthChecksScheduler.service.js': '[SuperBackend][Health]',
|
|
26
|
+
'consoleManager.service.js': '[SuperBackend][Console]',
|
|
27
|
+
'middleware': '[SuperBackend][Core]',
|
|
28
|
+
'middleware.js': '[SuperBackend][Core]'
|
|
29
|
+
};
|
|
30
|
+
|
|
16
31
|
let isActive = false;
|
|
17
32
|
let previousConsole = null;
|
|
18
33
|
let isHandling = false;
|
|
@@ -41,19 +56,9 @@ function normalizeMessage(message) {
|
|
|
41
56
|
.slice(0, 500);
|
|
42
57
|
}
|
|
43
58
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const lines = String(stack).split("\n").slice(3, 6);
|
|
48
|
-
for (const line of lines) {
|
|
49
|
-
const match = line.match(/at\s+(?:(.+?)\s+)?\(?(.+?):(\d+):(\d+)\)?/);
|
|
50
|
-
if (match) {
|
|
51
|
-
const fn = match[1] || "<anonymous>";
|
|
52
|
-
const file = match[2].split("/").pop();
|
|
53
|
-
return `${fn}@${file}:${match[3]}`;
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
return "";
|
|
59
|
+
// Simplified approach - no stack trace parsing needed
|
|
60
|
+
function getModulePrefix() {
|
|
61
|
+
return currentModulePrefix;
|
|
57
62
|
}
|
|
58
63
|
|
|
59
64
|
function computeHash({ method, messageTemplate, topFrame }) {
|
|
@@ -377,16 +382,26 @@ let configFromMemory = null;
|
|
|
377
382
|
function handleConsoleCall(method, args, stack) {
|
|
378
383
|
const message = buildMessageFromArgs(args);
|
|
379
384
|
const messageTemplate = normalizeMessage(message);
|
|
380
|
-
|
|
385
|
+
// Use empty string for topFrame since we're not using stack trace parsing
|
|
386
|
+
const topFrame = "";
|
|
381
387
|
const hash = computeHash({ method, messageTemplate, topFrame });
|
|
382
388
|
|
|
389
|
+
// Get the current module prefix (simplified approach)
|
|
390
|
+
const prefix = getModulePrefix();
|
|
391
|
+
|
|
392
|
+
// Add prefix to args if prefix exists
|
|
393
|
+
let prefixedArgs = args;
|
|
394
|
+
if (prefix) {
|
|
395
|
+
prefixedArgs = [prefix, ...args];
|
|
396
|
+
}
|
|
397
|
+
|
|
383
398
|
let entryFromMemory = memoryEntries.get(hash);
|
|
384
399
|
|
|
385
400
|
asyncUpdate();
|
|
386
401
|
|
|
387
402
|
if (!configFromMemory && !entryFromMemory) {
|
|
388
403
|
// First pass - always log and wait for async update to complete
|
|
389
|
-
previousConsole[method](...
|
|
404
|
+
previousConsole[method](...prefixedArgs);
|
|
390
405
|
return;
|
|
391
406
|
}
|
|
392
407
|
|
|
@@ -396,7 +411,7 @@ function handleConsoleCall(method, args, stack) {
|
|
|
396
411
|
: configFromMemory?.defaultEntryEnabled !== false;
|
|
397
412
|
|
|
398
413
|
if (isEnabled) {
|
|
399
|
-
previousConsole[method](...
|
|
414
|
+
previousConsole[method](...prefixedArgs);
|
|
400
415
|
} else {
|
|
401
416
|
// Entry is disabled - suppress stdout but still capture error aggregation for errors
|
|
402
417
|
if (method === "error") {
|
|
@@ -568,11 +583,30 @@ async function ensureRetentionCron() {
|
|
|
568
583
|
|
|
569
584
|
const consoleManager = {
|
|
570
585
|
getConsole:()=>console,
|
|
586
|
+
|
|
587
|
+
// New method to set module prefix
|
|
588
|
+
setModulePrefix(moduleName) {
|
|
589
|
+
// If moduleName is already a prefix, use it directly
|
|
590
|
+
if (moduleName.startsWith('[') && moduleName.endsWith(']')) {
|
|
591
|
+
currentModulePrefix = moduleName;
|
|
592
|
+
} else {
|
|
593
|
+
// Otherwise look it up in the mapping
|
|
594
|
+
currentModulePrefix = MODULE_PREFIXES[moduleName] || '';
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
|
|
598
|
+
// Get current module prefix
|
|
599
|
+
getModulePrefix() {
|
|
600
|
+
return currentModulePrefix;
|
|
601
|
+
},
|
|
602
|
+
|
|
571
603
|
init() {
|
|
572
604
|
if (isActive) return;
|
|
573
605
|
if (isHandling) return;
|
|
574
606
|
|
|
575
|
-
|
|
607
|
+
// Use the truly original console from consoleOverride if available
|
|
608
|
+
// Otherwise fall back to current console
|
|
609
|
+
previousConsole = consoleOverride.TRULY_ORIGINAL_CONSOLE || { ...console };
|
|
576
610
|
|
|
577
611
|
METHODS.forEach((method) => {
|
|
578
612
|
console[method] = (...args) => {
|
|
@@ -697,4 +731,8 @@ const consoleManager = {
|
|
|
697
731
|
},
|
|
698
732
|
};
|
|
699
733
|
|
|
700
|
-
module.exports =
|
|
734
|
+
module.exports = {
|
|
735
|
+
consoleManager,
|
|
736
|
+
MODULE_PREFIXES,
|
|
737
|
+
handleConsoleCall
|
|
738
|
+
};
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
const crypto = require('crypto');
|
|
2
|
+
const mongoose = require('mongoose');
|
|
3
|
+
|
|
4
|
+
const Experiment = require('../models/Experiment');
|
|
5
|
+
const ExperimentAssignment = require('../models/ExperimentAssignment');
|
|
6
|
+
const ExperimentEvent = require('../models/ExperimentEvent');
|
|
7
|
+
|
|
8
|
+
const cacheLayer = require('./cacheLayer.service');
|
|
9
|
+
const jsonConfigsService = require('./jsonConfigs.service');
|
|
10
|
+
|
|
11
|
+
function normalizeStr(v) {
|
|
12
|
+
return String(v || '').trim();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function normalizeOrgId(orgId) {
|
|
16
|
+
if (orgId === null || orgId === undefined || orgId === '') return null;
|
|
17
|
+
const str = String(orgId);
|
|
18
|
+
if (!mongoose.Types.ObjectId.isValid(str)) {
|
|
19
|
+
const err = new Error('Invalid orgId');
|
|
20
|
+
err.code = 'VALIDATION';
|
|
21
|
+
throw err;
|
|
22
|
+
}
|
|
23
|
+
return new mongoose.Types.ObjectId(str);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function normalizeExperimentCode(code) {
|
|
27
|
+
const c = normalizeStr(code);
|
|
28
|
+
if (!c) {
|
|
29
|
+
const err = new Error('experiment code is required');
|
|
30
|
+
err.code = 'VALIDATION';
|
|
31
|
+
throw err;
|
|
32
|
+
}
|
|
33
|
+
return c;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function normalizeSubjectId(subjectId) {
|
|
37
|
+
const s = normalizeStr(subjectId);
|
|
38
|
+
if (!s) {
|
|
39
|
+
const err = new Error('subjectId is required');
|
|
40
|
+
err.code = 'VALIDATION';
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
return s;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function computeSubjectKey({ orgId, subjectId }) {
|
|
47
|
+
const sid = normalizeSubjectId(subjectId);
|
|
48
|
+
const oid = orgId ? String(orgId) : 'global';
|
|
49
|
+
return `org:${oid}:subject:${sid}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function computeBucketInt(input, max) {
|
|
53
|
+
const hash = crypto.createHash('sha256').update(String(input), 'utf8').digest('hex');
|
|
54
|
+
const int = parseInt(hash.slice(0, 8), 16);
|
|
55
|
+
return max <= 0 ? 0 : int % max;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function pickWeightedVariant({ experiment, subjectKey }) {
|
|
59
|
+
const variants = Array.isArray(experiment?.variants) ? experiment.variants : [];
|
|
60
|
+
const eligible = variants
|
|
61
|
+
.map((v) => ({
|
|
62
|
+
key: normalizeStr(v?.key),
|
|
63
|
+
weight: Number(v?.weight || 0) || 0,
|
|
64
|
+
configSlug: normalizeStr(v?.configSlug),
|
|
65
|
+
}))
|
|
66
|
+
.filter((v) => v.key && v.weight > 0);
|
|
67
|
+
|
|
68
|
+
if (!eligible.length) {
|
|
69
|
+
const err = new Error('Experiment has no weighted variants');
|
|
70
|
+
err.code = 'VALIDATION';
|
|
71
|
+
throw err;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const total = eligible.reduce((acc, v) => acc + v.weight, 0);
|
|
75
|
+
const salt = normalizeStr(experiment?.assignment?.salt) || String(experiment?._id || '');
|
|
76
|
+
const pos = computeBucketInt(`${salt}:${subjectKey}`, total);
|
|
77
|
+
|
|
78
|
+
let cursor = 0;
|
|
79
|
+
for (const v of eligible) {
|
|
80
|
+
cursor += v.weight;
|
|
81
|
+
if (pos < cursor) return v;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return eligible[eligible.length - 1];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function getExperimentByCode({ orgId, code }) {
|
|
88
|
+
const c = normalizeExperimentCode(code);
|
|
89
|
+
const oid = orgId ? normalizeOrgId(orgId) : null;
|
|
90
|
+
|
|
91
|
+
const doc = await Experiment.findOne({ organizationId: oid, code: c }).lean();
|
|
92
|
+
if (doc) return doc;
|
|
93
|
+
|
|
94
|
+
if (oid) {
|
|
95
|
+
const globalDoc = await Experiment.findOne({ organizationId: null, code: c }).lean();
|
|
96
|
+
if (globalDoc) return globalDoc;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const err = new Error('Experiment not found');
|
|
100
|
+
err.code = 'NOT_FOUND';
|
|
101
|
+
throw err;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function resolveVariantConfig(variant) {
|
|
105
|
+
const slug = normalizeStr(variant?.configSlug);
|
|
106
|
+
if (!slug) return null;
|
|
107
|
+
return jsonConfigsService.getJsonConfigValueBySlug(slug);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function getOrCreateAssignment({ orgId, experimentCode, subjectId, context }) {
|
|
111
|
+
const exp = await getExperimentByCode({ orgId, code: experimentCode });
|
|
112
|
+
const effectiveOrgId = exp.organizationId ? String(exp.organizationId) : null;
|
|
113
|
+
const subjectKey = computeSubjectKey({ orgId: effectiveOrgId || orgId || null, subjectId });
|
|
114
|
+
|
|
115
|
+
const cacheKey = `${String(exp._id)}:${subjectKey}`;
|
|
116
|
+
const cached = await cacheLayer.get(cacheKey, { namespace: 'experiments.assignments' });
|
|
117
|
+
if (cached && cached.variantKey) return { experiment: exp, assignment: cached };
|
|
118
|
+
|
|
119
|
+
const existing = await ExperimentAssignment.findOne({ experimentId: exp._id, subjectKey }).lean();
|
|
120
|
+
if (existing) {
|
|
121
|
+
const assignment = {
|
|
122
|
+
experimentId: String(existing.experimentId),
|
|
123
|
+
organizationId: existing.organizationId ? String(existing.organizationId) : null,
|
|
124
|
+
subjectKey: existing.subjectKey,
|
|
125
|
+
variantKey: existing.variantKey,
|
|
126
|
+
assignedAt: existing.assignedAt,
|
|
127
|
+
context: existing.context || {},
|
|
128
|
+
};
|
|
129
|
+
await cacheLayer.set(cacheKey, assignment, { namespace: 'experiments.assignments', ttlSeconds: 60 });
|
|
130
|
+
return { experiment: exp, assignment };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (exp.status !== 'running' && exp.status !== 'completed') {
|
|
134
|
+
const err = new Error('Experiment is not active');
|
|
135
|
+
err.code = 'CONFLICT';
|
|
136
|
+
throw err;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const picked = pickWeightedVariant({ experiment: exp, subjectKey });
|
|
140
|
+
|
|
141
|
+
const created = await ExperimentAssignment.create({
|
|
142
|
+
experimentId: exp._id,
|
|
143
|
+
organizationId: exp.organizationId || null,
|
|
144
|
+
subjectKey,
|
|
145
|
+
variantKey: picked.key,
|
|
146
|
+
assignedAt: new Date(),
|
|
147
|
+
context: context && typeof context === 'object' ? context : {},
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const assignment = {
|
|
151
|
+
experimentId: String(created.experimentId),
|
|
152
|
+
organizationId: created.organizationId ? String(created.organizationId) : null,
|
|
153
|
+
subjectKey: created.subjectKey,
|
|
154
|
+
variantKey: created.variantKey,
|
|
155
|
+
assignedAt: created.assignedAt,
|
|
156
|
+
context: created.context || {},
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
await cacheLayer.set(cacheKey, assignment, { namespace: 'experiments.assignments', ttlSeconds: 60 });
|
|
160
|
+
return { experiment: exp, assignment };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function ingestEvents({ orgId, experimentCode, subjectId, events }) {
|
|
164
|
+
const exp = await getExperimentByCode({ orgId, code: experimentCode });
|
|
165
|
+
|
|
166
|
+
const effectiveOrgId = exp.organizationId ? String(exp.organizationId) : null;
|
|
167
|
+
const subjectKey = computeSubjectKey({ orgId: effectiveOrgId || orgId || null, subjectId });
|
|
168
|
+
|
|
169
|
+
const list = Array.isArray(events) ? events : [];
|
|
170
|
+
if (!list.length) {
|
|
171
|
+
const err = new Error('events[] is required');
|
|
172
|
+
err.code = 'VALIDATION';
|
|
173
|
+
throw err;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const variantKeys = new Set((exp.variants || []).map((v) => String(v?.key || '').trim()).filter(Boolean));
|
|
177
|
+
|
|
178
|
+
const now = new Date();
|
|
179
|
+
const docs = [];
|
|
180
|
+
for (const e of list) {
|
|
181
|
+
if (!e || typeof e !== 'object') continue;
|
|
182
|
+
|
|
183
|
+
const eventKey = normalizeStr(e.eventKey);
|
|
184
|
+
if (!eventKey) {
|
|
185
|
+
const err = new Error('eventKey is required');
|
|
186
|
+
err.code = 'VALIDATION';
|
|
187
|
+
throw err;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const ts = e.ts ? new Date(e.ts) : now;
|
|
191
|
+
if (!Number.isFinite(ts.getTime())) {
|
|
192
|
+
const err = new Error('Invalid ts');
|
|
193
|
+
err.code = 'VALIDATION';
|
|
194
|
+
throw err;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
let variantKey = normalizeStr(e.variantKey);
|
|
198
|
+
if (!variantKey) {
|
|
199
|
+
const { assignment } = await getOrCreateAssignment({ orgId, experimentCode, subjectId, context: null });
|
|
200
|
+
variantKey = assignment.variantKey;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (!variantKeys.has(variantKey)) {
|
|
204
|
+
const err = new Error('Invalid variantKey');
|
|
205
|
+
err.code = 'VALIDATION';
|
|
206
|
+
throw err;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const value = e.value === undefined ? 1 : Number(e.value);
|
|
210
|
+
if (!Number.isFinite(value)) {
|
|
211
|
+
const err = new Error('Invalid value');
|
|
212
|
+
err.code = 'VALIDATION';
|
|
213
|
+
throw err;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
docs.push({
|
|
217
|
+
experimentId: exp._id,
|
|
218
|
+
organizationId: exp.organizationId || null,
|
|
219
|
+
subjectKey,
|
|
220
|
+
variantKey,
|
|
221
|
+
eventKey,
|
|
222
|
+
value,
|
|
223
|
+
ts,
|
|
224
|
+
meta: e.meta && typeof e.meta === 'object' ? e.meta : {},
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
if (!docs.length) {
|
|
229
|
+
const err = new Error('No valid events provided');
|
|
230
|
+
err.code = 'VALIDATION';
|
|
231
|
+
throw err;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const inserted = await ExperimentEvent.insertMany(docs, { ordered: false });
|
|
235
|
+
return { insertedCount: Array.isArray(inserted) ? inserted.length : 0 };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async function getWinnerSnapshot({ orgId, experimentCode }) {
|
|
239
|
+
const exp = await getExperimentByCode({ orgId, code: experimentCode });
|
|
240
|
+
|
|
241
|
+
const cacheKey = String(exp._id);
|
|
242
|
+
const cached = await cacheLayer.get(cacheKey, { namespace: 'experiments.winner' });
|
|
243
|
+
if (cached && typeof cached === 'object') return { experiment: exp, snapshot: cached };
|
|
244
|
+
|
|
245
|
+
const snapshot = {
|
|
246
|
+
experimentId: String(exp._id),
|
|
247
|
+
organizationId: exp.organizationId ? String(exp.organizationId) : null,
|
|
248
|
+
code: exp.code,
|
|
249
|
+
status: exp.status,
|
|
250
|
+
winnerVariantKey: exp.winnerVariantKey || null,
|
|
251
|
+
winnerDecidedAt: exp.winnerDecidedAt || null,
|
|
252
|
+
winnerReason: exp.winnerReason || null,
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
await cacheLayer.set(cacheKey, snapshot, { namespace: 'experiments.winner', ttlSeconds: 30 });
|
|
256
|
+
return { experiment: exp, snapshot };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async function clearExperimentCaches(experimentId) {
|
|
260
|
+
const id = String(experimentId || '').trim();
|
|
261
|
+
if (!id) return;
|
|
262
|
+
await cacheLayer.delete(id, { namespace: 'experiments.winner' }).catch(() => {});
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
module.exports = {
|
|
266
|
+
computeSubjectKey,
|
|
267
|
+
getExperimentByCode,
|
|
268
|
+
resolveVariantConfig,
|
|
269
|
+
getOrCreateAssignment,
|
|
270
|
+
ingestEvents,
|
|
271
|
+
getWinnerSnapshot,
|
|
272
|
+
clearExperimentCaches,
|
|
273
|
+
};
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
const mongoose = require('mongoose');
|
|
2
|
+
|
|
3
|
+
const Experiment = require('../models/Experiment');
|
|
4
|
+
const ExperimentEvent = require('../models/ExperimentEvent');
|
|
5
|
+
const ExperimentMetricBucket = require('../models/ExperimentMetricBucket');
|
|
6
|
+
|
|
7
|
+
const experimentsService = require('./experiments.service');
|
|
8
|
+
const { broadcastWinnerChanged } = require('./experimentsWs.service');
|
|
9
|
+
const webhookService = require('./webhook.service');
|
|
10
|
+
|
|
11
|
+
function normalizeStr(v) {
|
|
12
|
+
return String(v || '').trim();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function floorToBucket(date, bucketMs) {
|
|
16
|
+
const t = new Date(date).getTime();
|
|
17
|
+
const ms = Number(bucketMs || 0) || 0;
|
|
18
|
+
if (!Number.isFinite(t) || ms <= 0) return null;
|
|
19
|
+
return new Date(Math.floor(t / ms) * ms);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function resolveMetricKeys(exp) {
|
|
23
|
+
const defs = [];
|
|
24
|
+
if (exp?.primaryMetric) defs.push(exp.primaryMetric);
|
|
25
|
+
for (const m of exp?.secondaryMetrics || []) defs.push(m);
|
|
26
|
+
|
|
27
|
+
const keys = new Set();
|
|
28
|
+
for (const d of defs) {
|
|
29
|
+
const kind = normalizeStr(d?.kind);
|
|
30
|
+
const key = normalizeStr(d?.key);
|
|
31
|
+
if (!key) continue;
|
|
32
|
+
|
|
33
|
+
if (kind === 'rate') {
|
|
34
|
+
const num = normalizeStr(d?.numeratorEventKey);
|
|
35
|
+
const den = normalizeStr(d?.denominatorEventKey);
|
|
36
|
+
if (num) keys.add(num);
|
|
37
|
+
if (den) keys.add(den);
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
keys.add(key);
|
|
42
|
+
}
|
|
43
|
+
return Array.from(keys);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function aggregateExperiment({ experimentId, bucketMs, start, end }) {
|
|
47
|
+
const exp = await Experiment.findById(experimentId).lean();
|
|
48
|
+
if (!exp) {
|
|
49
|
+
const err = new Error('Experiment not found');
|
|
50
|
+
err.code = 'NOT_FOUND';
|
|
51
|
+
throw err;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const metricEventKeys = resolveMetricKeys(exp);
|
|
55
|
+
if (!metricEventKeys.length) {
|
|
56
|
+
return { experimentId: String(exp._id), aggregated: 0 };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const startAt = start ? new Date(start) : (exp.startedAt ? new Date(exp.startedAt) : new Date(Date.now() - 24 * 60 * 60 * 1000));
|
|
60
|
+
const endAt = end ? new Date(end) : new Date();
|
|
61
|
+
|
|
62
|
+
const bucket = Number(bucketMs || 0) || 3600000;
|
|
63
|
+
|
|
64
|
+
const pipeline = [
|
|
65
|
+
{
|
|
66
|
+
$match: {
|
|
67
|
+
experimentId: new mongoose.Types.ObjectId(String(exp._id)),
|
|
68
|
+
ts: { $gte: startAt, $lte: endAt },
|
|
69
|
+
eventKey: { $in: metricEventKeys },
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
$addFields: {
|
|
74
|
+
bucketStart: {
|
|
75
|
+
$toDate: {
|
|
76
|
+
$multiply: [
|
|
77
|
+
{ $floor: { $divide: [{ $toLong: '$ts' }, bucket] } },
|
|
78
|
+
bucket,
|
|
79
|
+
],
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
$group: {
|
|
86
|
+
_id: {
|
|
87
|
+
variantKey: '$variantKey',
|
|
88
|
+
metricKey: '$eventKey',
|
|
89
|
+
bucketStart: '$bucketStart',
|
|
90
|
+
},
|
|
91
|
+
count: { $sum: 1 },
|
|
92
|
+
sum: { $sum: '$value' },
|
|
93
|
+
sumSq: { $sum: { $multiply: ['$value', '$value'] } },
|
|
94
|
+
min: { $min: '$value' },
|
|
95
|
+
max: { $max: '$value' },
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
];
|
|
99
|
+
|
|
100
|
+
const rows = await ExperimentEvent.aggregate(pipeline);
|
|
101
|
+
|
|
102
|
+
let aggregated = 0;
|
|
103
|
+
for (const r of rows || []) {
|
|
104
|
+
if (!r || !r._id) continue;
|
|
105
|
+
|
|
106
|
+
await ExperimentMetricBucket.updateOne(
|
|
107
|
+
{
|
|
108
|
+
experimentId: exp._id,
|
|
109
|
+
organizationId: exp.organizationId || null,
|
|
110
|
+
variantKey: String(r._id.variantKey),
|
|
111
|
+
metricKey: String(r._id.metricKey),
|
|
112
|
+
bucketStart: new Date(r._id.bucketStart),
|
|
113
|
+
bucketMs: bucket,
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
$set: {
|
|
117
|
+
count: Number(r.count || 0) || 0,
|
|
118
|
+
sum: Number(r.sum || 0) || 0,
|
|
119
|
+
sumSq: Number(r.sumSq || 0) || 0,
|
|
120
|
+
min: r.min === undefined ? null : r.min,
|
|
121
|
+
max: r.max === undefined ? null : r.max,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
{ upsert: true },
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
aggregated += 1;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { experimentId: String(exp._id), aggregated };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function computeTotalsForMetric({ experimentId, variantKey, metricKey, startAt }) {
|
|
134
|
+
const q = {
|
|
135
|
+
experimentId: new mongoose.Types.ObjectId(String(experimentId)),
|
|
136
|
+
variantKey: String(variantKey),
|
|
137
|
+
metricKey: String(metricKey),
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
if (startAt) {
|
|
141
|
+
q.bucketStart = { $gte: new Date(startAt) };
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const rows = await ExperimentMetricBucket.find(q).select('count sum').lean();
|
|
145
|
+
|
|
146
|
+
let count = 0;
|
|
147
|
+
let sum = 0;
|
|
148
|
+
for (const r of rows || []) {
|
|
149
|
+
count += Number(r.count || 0) || 0;
|
|
150
|
+
sum += Number(r.sum || 0) || 0;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return { count, sum };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async function evaluateWinner({ experimentId }) {
|
|
157
|
+
const exp = await Experiment.findById(experimentId);
|
|
158
|
+
if (!exp) {
|
|
159
|
+
const err = new Error('Experiment not found');
|
|
160
|
+
err.code = 'NOT_FOUND';
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (exp.winnerPolicy?.mode !== 'automatic') {
|
|
165
|
+
return { decided: false, reason: 'manual_mode' };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (exp.winnerVariantKey && exp.winnerDecidedAt) {
|
|
169
|
+
return { decided: true, reason: 'already_decided', winnerVariantKey: exp.winnerVariantKey };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const pickAfterMs = Number(exp.winnerPolicy?.pickAfterMs || 0) || 0;
|
|
173
|
+
const startedAt = exp.startedAt ? new Date(exp.startedAt) : null;
|
|
174
|
+
if (!startedAt) {
|
|
175
|
+
return { decided: false, reason: 'missing_startedAt' };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (pickAfterMs > 0 && Date.now() - startedAt.getTime() < pickAfterMs) {
|
|
179
|
+
return { decided: false, reason: 'too_early' };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const primary = exp.primaryMetric || {};
|
|
183
|
+
const kind = normalizeStr(primary.kind) || 'count';
|
|
184
|
+
|
|
185
|
+
const variants = (exp.variants || []).map((v) => normalizeStr(v?.key)).filter(Boolean);
|
|
186
|
+
if (!variants.length) {
|
|
187
|
+
return { decided: false, reason: 'no_variants' };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const evalStart = startedAt;
|
|
191
|
+
|
|
192
|
+
const scores = [];
|
|
193
|
+
for (const variantKey of variants) {
|
|
194
|
+
if (kind === 'rate') {
|
|
195
|
+
const numeratorKey = normalizeStr(primary.numeratorEventKey);
|
|
196
|
+
const denominatorKey = normalizeStr(primary.denominatorEventKey);
|
|
197
|
+
|
|
198
|
+
if (!numeratorKey || !denominatorKey) {
|
|
199
|
+
return { decided: false, reason: 'invalid_primary_rate_metric' };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const num = await computeTotalsForMetric({ experimentId: exp._id, variantKey, metricKey: numeratorKey, startAt: evalStart });
|
|
203
|
+
const den = await computeTotalsForMetric({ experimentId: exp._id, variantKey, metricKey: denominatorKey, startAt: evalStart });
|
|
204
|
+
|
|
205
|
+
const conversions = num.sum;
|
|
206
|
+
const exposures = den.sum;
|
|
207
|
+
|
|
208
|
+
const score = exposures > 0 ? conversions / exposures : 0;
|
|
209
|
+
scores.push({ variantKey, score, conversions, exposures });
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const metricKey = normalizeStr(primary.key);
|
|
214
|
+
if (!metricKey) {
|
|
215
|
+
return { decided: false, reason: 'invalid_primary_metric' };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const totals = await computeTotalsForMetric({ experimentId: exp._id, variantKey, metricKey, startAt: evalStart });
|
|
219
|
+
|
|
220
|
+
let score = 0;
|
|
221
|
+
if (kind === 'count') score = totals.count;
|
|
222
|
+
else if (kind === 'sum') score = totals.sum;
|
|
223
|
+
else if (kind === 'avg') score = totals.count > 0 ? totals.sum / totals.count : 0;
|
|
224
|
+
else score = totals.sum;
|
|
225
|
+
|
|
226
|
+
scores.push({ variantKey, score, count: totals.count, sum: totals.sum });
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const minAssignments = Number(exp.winnerPolicy?.minAssignments || 0) || 0;
|
|
230
|
+
const minExposures = Number(exp.winnerPolicy?.minExposures || 0) || 0;
|
|
231
|
+
const minConversions = Number(exp.winnerPolicy?.minConversions || 0) || 0;
|
|
232
|
+
|
|
233
|
+
if (kind === 'rate') {
|
|
234
|
+
const anyOk = scores.some((s) => (Number(s.exposures || 0) || 0) >= minExposures && (Number(s.conversions || 0) || 0) >= minConversions);
|
|
235
|
+
if (!anyOk) return { decided: false, reason: 'insufficient_data' };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (minAssignments > 0) {
|
|
239
|
+
// Placeholder: assignment counts could be derived from assignments collection.
|
|
240
|
+
// We enforce it later once we implement an efficient counter.
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const objective = normalizeStr(primary.objective) || 'maximize';
|
|
244
|
+
const sorted = [...scores].sort((a, b) => (objective === 'minimize' ? a.score - b.score : b.score - a.score));
|
|
245
|
+
const winner = sorted[0];
|
|
246
|
+
if (!winner) return { decided: false, reason: 'no_scores' };
|
|
247
|
+
|
|
248
|
+
const override = normalizeStr(exp.winnerPolicy?.overrideWinnerVariantKey);
|
|
249
|
+
const winnerKey = override || winner.variantKey;
|
|
250
|
+
|
|
251
|
+
exp.winnerVariantKey = winnerKey;
|
|
252
|
+
exp.winnerDecidedAt = new Date();
|
|
253
|
+
exp.winnerReason = override ? 'manual_override' : `auto:${kind}:${objective}`;
|
|
254
|
+
exp.status = 'completed';
|
|
255
|
+
|
|
256
|
+
await exp.save();
|
|
257
|
+
|
|
258
|
+
await experimentsService.clearExperimentCaches(exp._id);
|
|
259
|
+
|
|
260
|
+
broadcastWinnerChanged({
|
|
261
|
+
experimentId: String(exp._id),
|
|
262
|
+
experimentCode: exp.code,
|
|
263
|
+
organizationId: exp.organizationId ? String(exp.organizationId) : null,
|
|
264
|
+
winnerVariantKey: exp.winnerVariantKey,
|
|
265
|
+
decidedAt: exp.winnerDecidedAt,
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
if (exp.organizationId) {
|
|
269
|
+
webhookService.emit(
|
|
270
|
+
'experiment.winner_changed',
|
|
271
|
+
{
|
|
272
|
+
experimentId: String(exp._id),
|
|
273
|
+
code: exp.code,
|
|
274
|
+
winnerVariantKey: exp.winnerVariantKey,
|
|
275
|
+
decidedAt: exp.winnerDecidedAt,
|
|
276
|
+
},
|
|
277
|
+
String(exp.organizationId),
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
return { decided: true, reason: exp.winnerReason, winnerVariantKey: exp.winnerVariantKey, scores };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async function runAggregationAndWinner({ bucketMs, start, end } = {}) {
|
|
285
|
+
const now = new Date();
|
|
286
|
+
const bucket = Number(bucketMs || 0) || 3600000;
|
|
287
|
+
|
|
288
|
+
const startAt = start ? new Date(start) : new Date(now.getTime() - 6 * 60 * 60 * 1000);
|
|
289
|
+
const endAt = end ? new Date(end) : now;
|
|
290
|
+
|
|
291
|
+
const experiments = await Experiment.find({ status: { $in: ['running', 'completed'] } }).select('_id').lean();
|
|
292
|
+
|
|
293
|
+
const out = [];
|
|
294
|
+
for (const e of experiments || []) {
|
|
295
|
+
const res = await aggregateExperiment({ experimentId: e._id, bucketMs: bucket, start: startAt, end: endAt });
|
|
296
|
+
const win = await evaluateWinner({ experimentId: e._id }).catch((err) => ({ decided: false, reason: err.message }));
|
|
297
|
+
out.push({ experimentId: String(e._id), aggregated: res.aggregated, winner: win });
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return { range: { start: startAt.toISOString(), end: endAt.toISOString() }, bucketMs: bucket, items: out };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
module.exports = {
|
|
304
|
+
floorToBucket,
|
|
305
|
+
aggregateExperiment,
|
|
306
|
+
evaluateWinner,
|
|
307
|
+
runAggregationAndWinner,
|
|
308
|
+
};
|