kanmi-perf-revenue 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +347 -173
- package/dist/empirical/ab-testing.d.ts +83 -0
- package/dist/empirical/ab-testing.d.ts.map +1 -0
- package/dist/empirical/ab-testing.js +281 -0
- package/dist/empirical/ab-testing.js.map +1 -0
- package/dist/empirical/alerting.d.ts +85 -0
- package/dist/empirical/alerting.d.ts.map +1 -0
- package/dist/empirical/alerting.js +358 -0
- package/dist/empirical/alerting.js.map +1 -0
- package/dist/empirical/attribution.d.ts +80 -0
- package/dist/empirical/attribution.d.ts.map +1 -0
- package/dist/empirical/attribution.js +305 -0
- package/dist/empirical/attribution.js.map +1 -0
- package/dist/empirical/cohort.d.ts +75 -0
- package/dist/empirical/cohort.d.ts.map +1 -0
- package/dist/empirical/cohort.js +305 -0
- package/dist/empirical/cohort.js.map +1 -0
- package/dist/empirical/conversion-curve.d.ts +7 -10
- package/dist/empirical/conversion-curve.d.ts.map +1 -1
- package/dist/empirical/conversion-curve.js +37 -4
- package/dist/empirical/conversion-curve.js.map +1 -1
- package/dist/empirical/correlation.d.ts +91 -0
- package/dist/empirical/correlation.d.ts.map +1 -0
- package/dist/empirical/correlation.js +461 -0
- package/dist/empirical/correlation.js.map +1 -0
- package/dist/empirical/data-import.d.ts +22 -0
- package/dist/empirical/data-import.d.ts.map +1 -1
- package/dist/empirical/data-import.js +44 -0
- package/dist/empirical/data-import.js.map +1 -1
- package/dist/empirical/datadog-product-analytics.d.ts +192 -0
- package/dist/empirical/datadog-product-analytics.d.ts.map +1 -0
- package/dist/empirical/datadog-product-analytics.js +632 -0
- package/dist/empirical/datadog-product-analytics.js.map +1 -0
- package/dist/empirical/datadog-session-query.d.ts +32 -0
- package/dist/empirical/datadog-session-query.d.ts.map +1 -1
- package/dist/empirical/datadog-session-query.js +238 -17
- package/dist/empirical/datadog-session-query.js.map +1 -1
- package/dist/empirical/engagement-analysis.d.ts +112 -0
- package/dist/empirical/engagement-analysis.d.ts.map +1 -0
- package/dist/empirical/engagement-analysis.js +354 -0
- package/dist/empirical/engagement-analysis.js.map +1 -0
- package/dist/empirical/export.d.ts +75 -0
- package/dist/empirical/export.d.ts.map +1 -0
- package/dist/empirical/export.js +392 -0
- package/dist/empirical/export.js.map +1 -0
- package/dist/empirical/forecasting.d.ts +80 -0
- package/dist/empirical/forecasting.d.ts.map +1 -0
- package/dist/empirical/forecasting.js +287 -0
- package/dist/empirical/forecasting.js.map +1 -0
- package/dist/empirical/funnel.d.ts +66 -0
- package/dist/empirical/funnel.d.ts.map +1 -0
- package/dist/empirical/funnel.js +293 -0
- package/dist/empirical/funnel.js.map +1 -0
- package/dist/empirical/history.d.ts +198 -0
- package/dist/empirical/history.d.ts.map +1 -0
- package/dist/empirical/history.js +396 -0
- package/dist/empirical/history.js.map +1 -0
- package/dist/empirical/index.d.ts +41 -16
- package/dist/empirical/index.d.ts.map +1 -1
- package/dist/empirical/index.js +96 -13
- package/dist/empirical/index.js.map +1 -1
- package/dist/empirical/interactions.d.ts +89 -0
- package/dist/empirical/interactions.d.ts.map +1 -0
- package/dist/empirical/interactions.js +346 -0
- package/dist/empirical/interactions.js.map +1 -0
- package/dist/empirical/opportunity-calculator.d.ts +6 -18
- package/dist/empirical/opportunity-calculator.d.ts.map +1 -1
- package/dist/empirical/opportunity-calculator.js +19 -1
- package/dist/empirical/opportunity-calculator.js.map +1 -1
- package/dist/empirical/report.d.ts +3 -11
- package/dist/empirical/report.d.ts.map +1 -1
- package/dist/empirical/report.js +11 -7
- package/dist/empirical/report.js.map +1 -1
- package/dist/empirical/roi-calculator.d.ts +104 -0
- package/dist/empirical/roi-calculator.d.ts.map +1 -0
- package/dist/empirical/roi-calculator.js +403 -0
- package/dist/empirical/roi-calculator.js.map +1 -0
- package/dist/empirical/seasonality.d.ts +80 -0
- package/dist/empirical/seasonality.d.ts.map +1 -0
- package/dist/empirical/seasonality.js +340 -0
- package/dist/empirical/seasonality.js.map +1 -0
- package/dist/empirical/segmentation.d.ts +135 -0
- package/dist/empirical/segmentation.d.ts.map +1 -0
- package/dist/empirical/segmentation.js +379 -0
- package/dist/empirical/segmentation.js.map +1 -0
- package/dist/empirical/statistics.d.ts +118 -0
- package/dist/empirical/statistics.d.ts.map +1 -0
- package/dist/empirical/statistics.js +344 -0
- package/dist/empirical/statistics.js.map +1 -0
- package/dist/empirical/sweet-spot.d.ts +81 -0
- package/dist/empirical/sweet-spot.d.ts.map +1 -0
- package/dist/empirical/sweet-spot.js +198 -0
- package/dist/empirical/sweet-spot.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ab-testing.d.ts","sourceRoot":"","sources":["../../src/empirical/ab-testing.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAY9D,MAAM,WAAW,aAAa;IAC5B,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,8BAA8B;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,kBAAkB;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,kBAAkB;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,cAAc;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,SAAS,EAAE,aAAa,CAAC;IACzB,8BAA8B;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qDAAqD;IACrD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,aAAa,EAAE,OAAO,CAAC;IACvB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,qBAAqB,EAAE;QACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,+BAA+B;QAC/B,mBAAmB,EAAE,OAAO,CAAC;KAC9B,CAAC;IACF,qBAAqB;IACrB,cAAc,EAAE,gBAAgB,GAAG,cAAc,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACtF,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B,kBAAkB,EAAE;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,OAAO,CAAC;QACtB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;KACnC,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAMD;;GAEG;AACH,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,WAAW,EAAE,EACvB,YAAY,CAAC,EAAE,MAAM,GACpB,YAAY,GAAG,IAAI,CAyHrB;AAsED;;GAEG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,WAAW,EAAE,EACvB,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,YAAQ,EACrC,SAAS,EAAE,MAAM,GAChB,YAAY,CAwCd;AAMD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CA+EnE"}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A/B Test Integration
|
|
3
|
+
*
|
|
4
|
+
* Compare performance variants and analyze their impact on conversions.
|
|
5
|
+
* Integrates with experiment data to determine if performance changes
|
|
6
|
+
* drove conversion improvements.
|
|
7
|
+
*
|
|
8
|
+
* @author Kanmi Obasa <i@kanmiobasa.com>
|
|
9
|
+
*/
|
|
10
|
+
import { twoProportionZTest, calculateConfidenceInterval, calculateEffectSize, calculateRequiredSampleSize, } from './statistics.js';
|
|
11
|
+
// =============================================================================
|
|
12
|
+
// CORE ANALYSIS
|
|
13
|
+
// =============================================================================
|
|
14
|
+
/**
|
|
15
|
+
* Analyze an A/B test from session data.
|
|
16
|
+
*/
|
|
17
|
+
export function analyzeABTest(sessions, experimentId) {
|
|
18
|
+
// Filter to experiment sessions
|
|
19
|
+
const experimentSessions = experimentId
|
|
20
|
+
? sessions.filter(s => s.experiment_id === experimentId)
|
|
21
|
+
: sessions.filter(s => s.experiment_id);
|
|
22
|
+
if (experimentSessions.length < 100) {
|
|
23
|
+
return null; // Not enough data
|
|
24
|
+
}
|
|
25
|
+
// Get unique experiment ID
|
|
26
|
+
const expId = experimentId || experimentSessions[0].experiment_id || 'unknown';
|
|
27
|
+
// Group by variant
|
|
28
|
+
const variantMap = new Map();
|
|
29
|
+
for (const session of experimentSessions) {
|
|
30
|
+
const variant = session.experiment_variant || 'unknown';
|
|
31
|
+
if (!variantMap.has(variant)) {
|
|
32
|
+
variantMap.set(variant, []);
|
|
33
|
+
}
|
|
34
|
+
variantMap.get(variant).push(session);
|
|
35
|
+
}
|
|
36
|
+
// Build variant stats
|
|
37
|
+
const variants = [];
|
|
38
|
+
for (const [variantId, variantSessions] of variantMap) {
|
|
39
|
+
const conversions = variantSessions.filter(s => s.has_purchase).length;
|
|
40
|
+
const cvr = variantSessions.length > 0 ? conversions / variantSessions.length : 0;
|
|
41
|
+
const ci = calculateConfidenceInterval(conversions, variantSessions.length);
|
|
42
|
+
const lcpValues = variantSessions.map(s => s.lcp_ms).filter((v) => v !== null);
|
|
43
|
+
const inpValues = variantSessions.map(s => s.inp_ms).filter((v) => v !== null);
|
|
44
|
+
const sortedLcp = [...lcpValues].sort((a, b) => a - b);
|
|
45
|
+
variants.push({
|
|
46
|
+
variantId,
|
|
47
|
+
name: variantId === 'control' ? 'Control' : `Treatment: ${variantId}`,
|
|
48
|
+
sessions: variantSessions.length,
|
|
49
|
+
conversions,
|
|
50
|
+
cvr,
|
|
51
|
+
cvrCI: { lower: ci.lower, upper: ci.upper },
|
|
52
|
+
avgLcp: lcpValues.length > 0 ? lcpValues.reduce((a, b) => a + b, 0) / lcpValues.length : null,
|
|
53
|
+
avgInp: inpValues.length > 0 ? inpValues.reduce((a, b) => a + b, 0) / inpValues.length : null,
|
|
54
|
+
p75Lcp: sortedLcp.length > 0 ? sortedLcp[Math.floor(sortedLcp.length * 0.75)] : null,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
// Sort by sessions (largest first, assume first is control)
|
|
58
|
+
variants.sort((a, b) => b.sessions - a.sessions);
|
|
59
|
+
// Build comparisons (each treatment vs control)
|
|
60
|
+
const comparisons = [];
|
|
61
|
+
const control = variants.find(v => v.variantId === 'control') || variants[0];
|
|
62
|
+
for (const treatment of variants) {
|
|
63
|
+
if (treatment.variantId === control.variantId)
|
|
64
|
+
continue;
|
|
65
|
+
const comparison = compareVariants(control, treatment);
|
|
66
|
+
comparisons.push(comparison);
|
|
67
|
+
}
|
|
68
|
+
// Determine winner
|
|
69
|
+
let winner = null;
|
|
70
|
+
const significantWins = comparisons.filter(c => c.isSignificant && c.absoluteDifference > 0 && c.recommendation === 'ship_treatment');
|
|
71
|
+
if (significantWins.length > 0) {
|
|
72
|
+
winner = variants.find(v => v.variantId === significantWins[0].treatment.variantId) || null;
|
|
73
|
+
}
|
|
74
|
+
// Sample size analysis
|
|
75
|
+
const totalSamples = experimentSessions.length;
|
|
76
|
+
const overallCvr = experimentSessions.filter(s => s.has_purchase).length / totalSamples;
|
|
77
|
+
const sampleSizeRec = calculateRequiredSampleSize(overallCvr, 0.1); // Detect 10% relative change
|
|
78
|
+
const requiredSamples = sampleSizeRec.sample_size_per_group * 2; // Both groups
|
|
79
|
+
const isSufficient = totalSamples >= requiredSamples;
|
|
80
|
+
// Estimate days to significance
|
|
81
|
+
const dailySessions = totalSamples / 14; // Assume 2 weeks of data
|
|
82
|
+
const daysToSignificance = !isSufficient && dailySessions > 0
|
|
83
|
+
? Math.ceil((requiredSamples - totalSamples) / dailySessions)
|
|
84
|
+
: null;
|
|
85
|
+
// Generate insights
|
|
86
|
+
const insights = [];
|
|
87
|
+
if (winner) {
|
|
88
|
+
insights.push(`Winner: ${winner.name} with ${(winner.cvr * 100).toFixed(2)}% CVR`);
|
|
89
|
+
}
|
|
90
|
+
for (const comparison of comparisons) {
|
|
91
|
+
if (comparison.performanceComparison.performanceImproved && comparison.absoluteDifference > 0) {
|
|
92
|
+
insights.push(`Performance improvement correlated with ${(comparison.relativeDifference * 100).toFixed(1)}% CVR lift`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (!isSufficient) {
|
|
96
|
+
insights.push(`Need ${(requiredSamples - totalSamples).toLocaleString()} more samples for statistical power`);
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
experimentId: expId,
|
|
100
|
+
variants,
|
|
101
|
+
comparisons,
|
|
102
|
+
winner,
|
|
103
|
+
sampleSizeAnalysis: {
|
|
104
|
+
currentSamples: totalSamples,
|
|
105
|
+
requiredSamples,
|
|
106
|
+
isSufficient,
|
|
107
|
+
daysToSignificance,
|
|
108
|
+
},
|
|
109
|
+
insights,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Compare two variants.
|
|
114
|
+
*/
|
|
115
|
+
function compareVariants(control, treatment) {
|
|
116
|
+
const absoluteDifference = treatment.cvr - control.cvr;
|
|
117
|
+
const relativeDifference = control.cvr > 0 ? absoluteDifference / control.cvr : 0;
|
|
118
|
+
const test = twoProportionZTest('treatment', treatment.conversions, treatment.sessions, 'control', control.conversions, control.sessions);
|
|
119
|
+
const effectSize = calculateEffectSize(treatment.cvr, control.cvr);
|
|
120
|
+
// Performance comparison
|
|
121
|
+
const lcpDelta = treatment.avgLcp !== null && control.avgLcp !== null
|
|
122
|
+
? treatment.avgLcp - control.avgLcp
|
|
123
|
+
: null;
|
|
124
|
+
const inpDelta = treatment.avgInp !== null && control.avgInp !== null
|
|
125
|
+
? treatment.avgInp - control.avgInp
|
|
126
|
+
: null;
|
|
127
|
+
const performanceImproved = (lcpDelta !== null && lcpDelta < 0) ||
|
|
128
|
+
(inpDelta !== null && inpDelta < 0);
|
|
129
|
+
// Determine recommendation
|
|
130
|
+
let recommendation;
|
|
131
|
+
let rationale;
|
|
132
|
+
if (test.p_value < 0.05) {
|
|
133
|
+
if (absoluteDifference > 0) {
|
|
134
|
+
recommendation = 'ship_treatment';
|
|
135
|
+
rationale = `Treatment shows ${(relativeDifference * 100).toFixed(1)}% improvement with p=${test.p_value.toFixed(4)}`;
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
recommendation = 'keep_control';
|
|
139
|
+
rationale = `Control performs better with ${((-relativeDifference) * 100).toFixed(1)}% higher CVR`;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
else if (control.sessions + treatment.sessions < 1000) {
|
|
143
|
+
recommendation = 'need_more_data';
|
|
144
|
+
rationale = `Only ${control.sessions + treatment.sessions} samples - need more data for significance`;
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
recommendation = 'inconclusive';
|
|
148
|
+
rationale = `No significant difference detected (p=${test.p_value.toFixed(4)})`;
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
control,
|
|
152
|
+
treatment,
|
|
153
|
+
absoluteDifference,
|
|
154
|
+
relativeDifference,
|
|
155
|
+
pValue: test.p_value,
|
|
156
|
+
isSignificant: test.is_significant,
|
|
157
|
+
effectSize: effectSize.cohens_h,
|
|
158
|
+
performanceComparison: {
|
|
159
|
+
lcpDelta,
|
|
160
|
+
inpDelta,
|
|
161
|
+
performanceImproved,
|
|
162
|
+
},
|
|
163
|
+
recommendation,
|
|
164
|
+
rationale,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Create a synthetic A/B test by splitting sessions on performance.
|
|
169
|
+
*/
|
|
170
|
+
export function createPerformanceExperiment(sessions, metric = 'lcp', threshold) {
|
|
171
|
+
const getValue = (s) => {
|
|
172
|
+
switch (metric) {
|
|
173
|
+
case 'lcp': return s.lcp_ms;
|
|
174
|
+
case 'inp': return s.inp_ms;
|
|
175
|
+
case 'cls': return s.cls;
|
|
176
|
+
default: return null;
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
// Split sessions by threshold
|
|
180
|
+
const controlSessions = sessions.filter(s => {
|
|
181
|
+
const value = getValue(s);
|
|
182
|
+
return value !== null && value > threshold;
|
|
183
|
+
});
|
|
184
|
+
const treatmentSessions = sessions.filter(s => {
|
|
185
|
+
const value = getValue(s);
|
|
186
|
+
return value !== null && value <= threshold;
|
|
187
|
+
});
|
|
188
|
+
// Create synthetic experiment
|
|
189
|
+
const modifiedSessions = [
|
|
190
|
+
...controlSessions.map(s => ({ ...s, experiment_id: 'perf-experiment', experiment_variant: 'control' })),
|
|
191
|
+
...treatmentSessions.map(s => ({ ...s, experiment_id: 'perf-experiment', experiment_variant: 'fast' })),
|
|
192
|
+
];
|
|
193
|
+
return analyzeABTest(modifiedSessions, 'perf-experiment') || {
|
|
194
|
+
experimentId: 'perf-experiment',
|
|
195
|
+
variants: [],
|
|
196
|
+
comparisons: [],
|
|
197
|
+
winner: null,
|
|
198
|
+
sampleSizeAnalysis: {
|
|
199
|
+
currentSamples: 0,
|
|
200
|
+
requiredSamples: 1000,
|
|
201
|
+
isSufficient: false,
|
|
202
|
+
daysToSignificance: null,
|
|
203
|
+
},
|
|
204
|
+
insights: ['Insufficient data for analysis'],
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
// =============================================================================
|
|
208
|
+
// REPORT GENERATION
|
|
209
|
+
// =============================================================================
|
|
210
|
+
/**
|
|
211
|
+
* Generate markdown report for A/B test.
|
|
212
|
+
*/
|
|
213
|
+
export function generateABTestMarkdown(report) {
|
|
214
|
+
const lines = [];
|
|
215
|
+
lines.push('## A/B Test Analysis');
|
|
216
|
+
lines.push('');
|
|
217
|
+
lines.push(`**Experiment**: ${report.experimentId}`);
|
|
218
|
+
lines.push('');
|
|
219
|
+
// Winner banner
|
|
220
|
+
if (report.winner) {
|
|
221
|
+
lines.push(`### 🏆 Winner: ${report.winner.name}`);
|
|
222
|
+
lines.push('');
|
|
223
|
+
}
|
|
224
|
+
// Insights
|
|
225
|
+
if (report.insights.length > 0) {
|
|
226
|
+
lines.push('### Insights');
|
|
227
|
+
lines.push('');
|
|
228
|
+
for (const insight of report.insights) {
|
|
229
|
+
lines.push(`- ${insight}`);
|
|
230
|
+
}
|
|
231
|
+
lines.push('');
|
|
232
|
+
}
|
|
233
|
+
// Variant comparison
|
|
234
|
+
lines.push('### Variants');
|
|
235
|
+
lines.push('');
|
|
236
|
+
lines.push('| Variant | Sessions | Conversions | CVR | 95% CI | Avg LCP |');
|
|
237
|
+
lines.push('|---------|----------|-------------|-----|--------|---------|');
|
|
238
|
+
for (const variant of report.variants) {
|
|
239
|
+
const ci = `${(variant.cvrCI.lower * 100).toFixed(2)}%-${(variant.cvrCI.upper * 100).toFixed(2)}%`;
|
|
240
|
+
const lcp = variant.avgLcp ? `${variant.avgLcp.toFixed(0)}ms` : 'N/A';
|
|
241
|
+
lines.push(`| ${variant.name} | ${variant.sessions.toLocaleString()} | ${variant.conversions} | ${(variant.cvr * 100).toFixed(2)}% | ${ci} | ${lcp} |`);
|
|
242
|
+
}
|
|
243
|
+
lines.push('');
|
|
244
|
+
// Comparisons
|
|
245
|
+
if (report.comparisons.length > 0) {
|
|
246
|
+
lines.push('### Statistical Analysis');
|
|
247
|
+
lines.push('');
|
|
248
|
+
for (const comparison of report.comparisons) {
|
|
249
|
+
const icon = comparison.recommendation === 'ship_treatment' ? '✅' :
|
|
250
|
+
comparison.recommendation === 'keep_control' ? '❌' :
|
|
251
|
+
comparison.recommendation === 'need_more_data' ? '⏳' : '❓';
|
|
252
|
+
lines.push(`#### ${comparison.treatment.name} vs ${comparison.control.name}`);
|
|
253
|
+
lines.push('');
|
|
254
|
+
lines.push(`| Metric | Value |`);
|
|
255
|
+
lines.push(`|--------|-------|`);
|
|
256
|
+
lines.push(`| Absolute Difference | ${(comparison.absoluteDifference * 100).toFixed(3)}% |`);
|
|
257
|
+
lines.push(`| Relative Difference | ${(comparison.relativeDifference * 100).toFixed(1)}% |`);
|
|
258
|
+
lines.push(`| p-value | ${comparison.pValue.toFixed(4)} |`);
|
|
259
|
+
lines.push(`| Effect Size | ${comparison.effectSize.toFixed(3)} |`);
|
|
260
|
+
lines.push(`| Significant | ${comparison.isSignificant ? 'Yes' : 'No'} |`);
|
|
261
|
+
lines.push(`| Recommendation | ${icon} ${comparison.recommendation.replace(/_/g, ' ')} |`);
|
|
262
|
+
lines.push('');
|
|
263
|
+
lines.push(`**Rationale**: ${comparison.rationale}`);
|
|
264
|
+
lines.push('');
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
// Sample size
|
|
268
|
+
lines.push('### Sample Size Analysis');
|
|
269
|
+
lines.push('');
|
|
270
|
+
lines.push(`| Metric | Value |`);
|
|
271
|
+
lines.push(`|--------|-------|`);
|
|
272
|
+
lines.push(`| Current Samples | ${report.sampleSizeAnalysis.currentSamples.toLocaleString()} |`);
|
|
273
|
+
lines.push(`| Required Samples | ${report.sampleSizeAnalysis.requiredSamples.toLocaleString()} |`);
|
|
274
|
+
lines.push(`| Sufficient | ${report.sampleSizeAnalysis.isSufficient ? 'Yes' : 'No'} |`);
|
|
275
|
+
if (report.sampleSizeAnalysis.daysToSignificance !== null) {
|
|
276
|
+
lines.push(`| Days to Significance | ~${report.sampleSizeAnalysis.daysToSignificance} days |`);
|
|
277
|
+
}
|
|
278
|
+
lines.push('');
|
|
279
|
+
return lines.join('\n');
|
|
280
|
+
}
|
|
281
|
+
//# sourceMappingURL=ab-testing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ab-testing.js","sourceRoot":"","sources":["../../src/empirical/ab-testing.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,iBAAiB,CAAC;AAkEzB,gFAAgF;AAChF,gBAAgB;AAChB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAuB,EACvB,YAAqB;IAErB,gCAAgC;IAChC,MAAM,kBAAkB,GAAG,YAAY;QACrC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,YAAY,CAAC;QACxD,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IAE1C,IAAI,kBAAkB,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,CAAC,kBAAkB;IACjC,CAAC;IAED,2BAA2B;IAC3B,MAAM,KAAK,GAAG,YAAY,IAAI,kBAAkB,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,SAAS,CAAC;IAE/E,mBAAmB;IACnB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEpD,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB,IAAI,SAAS,CAAC;QACxD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC9B,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,sBAAsB;IACtB,MAAM,QAAQ,GAAoB,EAAE,CAAC;IAErC,KAAK,MAAM,CAAC,SAAS,EAAE,eAAe,CAAC,IAAI,UAAU,EAAE,CAAC;QACtD,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QACvE,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,EAAE,GAAG,2BAA2B,CAAC,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;QAE5E,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC5F,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAE5F,MAAM,SAAS,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEvD,QAAQ,CAAC,IAAI,CAAC;YACZ,SAAS;YACT,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,SAAS,EAAE;YACrE,QAAQ,EAAE,eAAe,CAAC,MAAM;YAChC,WAAW;YACX,GAAG;YACH,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE;YAC3C,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAC7F,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAC7F,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;SACrF,CAAC,CAAC;IACL,CAAC;IAED,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAEjD,gDAAgD;IAChD,MAAM,WAAW,GAAuB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;IAE7E,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE,CAAC;QACjC,IAAI,SAAS,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS;YAAE,SAAS;QAExD,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACvD,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/B,CAAC;IAED,mBAAmB;IACnB,IAAI,MAAM,GAAyB,IAAI,CAAC;IACxC,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CACxC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,kBAAkB,GAAG,CAAC,IAAI,CAAC,CAAC,cAAc,KAAK,gBAAgB,CAC1F,CAAC;IAEF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IAC9F,CAAC;IAED,uBAAuB;IACvB,MAAM,YAAY,GAAG,kBAAkB,CAAC,MAAM,CAAC;IAC/C,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC;IACxF,MAAM,aAAa,GAAG,2BAA2B,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,6BAA6B;IACjG,MAAM,eAAe,GAAG,aAAa,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,cAAc;IAC/E,MAAM,YAAY,GAAG,YAAY,IAAI,eAAe,CAAC;IAErD,gCAAgC;IAChC,MAAM,aAAa,GAAG,YAAY,GAAG,EAAE,CAAC,CAAC,yBAAyB;IAClE,MAAM,kBAAkB,GAAG,CAAC,YAAY,IAAI,aAAa,GAAG,CAAC;QAC3D,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,eAAe,GAAG,YAAY,CAAC,GAAG,aAAa,CAAC;QAC7D,CAAC,CAAC,IAAI,CAAC;IAET,oBAAoB;IACpB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,MAAM,EAAE,CAAC;QACX,QAAQ,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACrF,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,UAAU,CAAC,qBAAqB,CAAC,mBAAmB,IAAI,UAAU,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC9F,QAAQ,CAAC,IAAI,CACX,2CAA2C,CAAC,UAAU,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CACX,QAAQ,CAAC,eAAe,GAAG,YAAY,CAAC,CAAC,cAAc,EAAE,qCAAqC,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY,EAAE,KAAK;QACnB,QAAQ;QACR,WAAW;QACX,MAAM;QACN,kBAAkB,EAAE;YAClB,cAAc,EAAE,YAAY;YAC5B,eAAe;YACf,YAAY;YACZ,kBAAkB;SACnB;QACD,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAsB,EAAE,SAAwB;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACvD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,IAAI,GAAG,kBAAkB,CAC7B,WAAW,EACX,SAAS,CAAC,WAAW,EACrB,SAAS,CAAC,QAAQ,EAClB,SAAS,EACT,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,QAAQ,CACjB,CAAC;IAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAEnE,yBAAyB;IACzB,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;QACnE,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QACnC,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;QACnE,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QACnC,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,mBAAmB,GAAG,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;QAC7D,CAAC,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAEtC,2BAA2B;IAC3B,IAAI,cAAkD,CAAC;IACvD,IAAI,SAAiB,CAAC;IAEtB,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,CAAC;QACxB,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC3B,cAAc,GAAG,gBAAgB,CAAC;YAClC,SAAS,GAAG,mBAAmB,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACxH,CAAC;aAAM,CAAC;YACN,cAAc,GAAG,cAAc,CAAC;YAChC,SAAS,GAAG,gCAAgC,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC;QACrG,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,GAAG,IAAI,EAAE,CAAC;QACxD,cAAc,GAAG,gBAAgB,CAAC;QAClC,SAAS,GAAG,QAAQ,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,4CAA4C,CAAC;IACxG,CAAC;SAAM,CAAC;QACN,cAAc,GAAG,cAAc,CAAC;QAChC,SAAS,GAAG,yCAAyC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAClF,CAAC;IAED,OAAO;QACL,OAAO;QACP,SAAS;QACT,kBAAkB;QAClB,kBAAkB;QAClB,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,aAAa,EAAE,IAAI,CAAC,cAAc;QAClC,UAAU,EAAE,UAAU,CAAC,QAAQ;QAC/B,qBAAqB,EAAE;YACrB,QAAQ;YACR,QAAQ;YACR,mBAAmB;SACpB;QACD,cAAc;QACd,SAAS;KACV,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAAuB,EACvB,SAAgC,KAAK,EACrC,SAAiB;IAEjB,MAAM,QAAQ,GAAG,CAAC,CAAc,EAAiB,EAAE;QACjD,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YAC5B,KAAK,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YAC5B,KAAK,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC;YACzB,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC;QACvB,CAAC;IACH,CAAC,CAAC;IAEF,8BAA8B;IAC9B,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,IAAI,SAAS,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,8BAA8B;IAC9B,MAAM,gBAAgB,GAAG;QACvB,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC;QACxG,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC;KACxG,CAAC;IAEF,OAAO,aAAa,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,IAAI;QAC3D,YAAY,EAAE,iBAAiB;QAC/B,QAAQ,EAAE,EAAE;QACZ,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,IAAI;QACZ,kBAAkB,EAAE;YAClB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,IAAI;YACrB,YAAY,EAAE,KAAK;YACnB,kBAAkB,EAAE,IAAI;SACzB;QACD,QAAQ,EAAE,CAAC,gCAAgC,CAAC;KAC7C,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAoB;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,WAAW;IACX,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;QAC7B,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,qBAAqB;IACrB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;IAE5E,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACnG,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;QACtE,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,OAAO,CAAC,WAAW,MAAM,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAC5I,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,cAAc;IACd,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,cAAc,KAAK,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACjE,UAAU,CAAC,cAAc,KAAK,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACpD,UAAU,CAAC,cAAc,KAAK,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAE7D,KAAK,CAAC,IAAI,CAAC,QAAQ,UAAU,CAAC,SAAS,CAAC,IAAI,OAAO,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,UAAU,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7F,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,UAAU,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7F,KAAK,CAAC,IAAI,CAAC,eAAe,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,mBAAmB,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,KAAK,CAAC,IAAI,CAAC,mBAAmB,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAC3E,KAAK,CAAC,IAAI,CAAC,sBAAsB,IAAI,IAAI,UAAU,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,UAAU,CAAC,SAAS,EAAE,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,kBAAkB,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,kBAAkB,CAAC,eAAe,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACnG,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACxF,IAAI,MAAM,CAAC,kBAAkB,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,6BAA6B,MAAM,CAAC,kBAAkB,CAAC,kBAAkB,SAAS,CAAC,CAAC;IACjG,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Alerting System
|
|
3
|
+
*
|
|
4
|
+
* Notify when performance metrics degrade or fall below thresholds.
|
|
5
|
+
* Provides configurable alerts and monitoring.
|
|
6
|
+
*
|
|
7
|
+
* @author Kanmi Obasa <i@kanmiobasa.com>
|
|
8
|
+
*/
|
|
9
|
+
import type { SessionData } from './datadog-session-query.js';
|
|
10
|
+
export type AlertSeverity = 'info' | 'warning' | 'critical';
|
|
11
|
+
export type AlertType = 'threshold' | 'regression' | 'anomaly' | 'opportunity';
|
|
12
|
+
export interface AlertThreshold {
|
|
13
|
+
metric: 'lcp' | 'inp' | 'cls' | 'fcp' | 'ttfb' | 'cvr' | 'bounce_rate';
|
|
14
|
+
warningThreshold: number;
|
|
15
|
+
criticalThreshold: number;
|
|
16
|
+
/** 'above' means alert if value > threshold, 'below' means alert if value < threshold */
|
|
17
|
+
direction: 'above' | 'below';
|
|
18
|
+
}
|
|
19
|
+
export interface Alert {
|
|
20
|
+
id: string;
|
|
21
|
+
type: AlertType;
|
|
22
|
+
severity: AlertSeverity;
|
|
23
|
+
metric: string;
|
|
24
|
+
currentValue: number;
|
|
25
|
+
threshold: number;
|
|
26
|
+
message: string;
|
|
27
|
+
timestamp: string;
|
|
28
|
+
context: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
export interface AlertConfig {
|
|
31
|
+
thresholds: AlertThreshold[];
|
|
32
|
+
/** Enable regression detection (compare to baseline) */
|
|
33
|
+
enableRegressionDetection: boolean;
|
|
34
|
+
/** Percentage change to trigger regression alert */
|
|
35
|
+
regressionThreshold: number;
|
|
36
|
+
/** Enable anomaly detection */
|
|
37
|
+
enableAnomalyDetection: boolean;
|
|
38
|
+
/** Standard deviations for anomaly */
|
|
39
|
+
anomalyStdDevThreshold: number;
|
|
40
|
+
}
|
|
41
|
+
export interface AlertReport {
|
|
42
|
+
alerts: Alert[];
|
|
43
|
+
summary: {
|
|
44
|
+
critical: number;
|
|
45
|
+
warning: number;
|
|
46
|
+
info: number;
|
|
47
|
+
total: number;
|
|
48
|
+
};
|
|
49
|
+
metrics: {
|
|
50
|
+
metric: string;
|
|
51
|
+
currentValue: number;
|
|
52
|
+
trend: 'improving' | 'stable' | 'degrading';
|
|
53
|
+
status: 'good' | 'warning' | 'critical';
|
|
54
|
+
}[];
|
|
55
|
+
}
|
|
56
|
+
export declare const DEFAULT_THRESHOLDS: AlertThreshold[];
|
|
57
|
+
export declare const DEFAULT_ALERT_CONFIG: AlertConfig;
|
|
58
|
+
/**
|
|
59
|
+
* Check sessions against thresholds and generate alerts.
|
|
60
|
+
*/
|
|
61
|
+
export declare function checkThresholds(sessions: SessionData[], config?: AlertConfig): Alert[];
|
|
62
|
+
/**
|
|
63
|
+
* Detect regressions compared to baseline.
|
|
64
|
+
*/
|
|
65
|
+
export declare function detectRegressions(currentSessions: SessionData[], baselineSessions: SessionData[], config?: AlertConfig): Alert[];
|
|
66
|
+
/**
|
|
67
|
+
* Detect anomalies using statistical methods.
|
|
68
|
+
*/
|
|
69
|
+
export declare function detectAnomalies(sessions: SessionData[], historicalMeans: Record<string, {
|
|
70
|
+
mean: number;
|
|
71
|
+
stdDev: number;
|
|
72
|
+
}>, config?: AlertConfig): Alert[];
|
|
73
|
+
/**
|
|
74
|
+
* Check for opportunities (positive alerts).
|
|
75
|
+
*/
|
|
76
|
+
export declare function detectOpportunities(sessions: SessionData[]): Alert[];
|
|
77
|
+
/**
|
|
78
|
+
* Generate comprehensive alert report.
|
|
79
|
+
*/
|
|
80
|
+
export declare function generateAlertReport(sessions: SessionData[], config?: AlertConfig, baseline?: SessionData[]): AlertReport;
|
|
81
|
+
/**
|
|
82
|
+
* Generate markdown report for alerts.
|
|
83
|
+
*/
|
|
84
|
+
export declare function generateAlertMarkdown(report: AlertReport): string;
|
|
85
|
+
//# sourceMappingURL=alerting.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"alerting.d.ts","sourceRoot":"","sources":["../../src/empirical/alerting.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAM9D,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAC5D,MAAM,MAAM,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,GAAG,aAAa,CAAC;AAE/E,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,aAAa,CAAC;IACvE,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yFAAyF;IACzF,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,wDAAwD;IACxD,yBAAyB,EAAE,OAAO,CAAC;IACnC,oDAAoD;IACpD,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+BAA+B;IAC/B,sBAAsB,EAAE,OAAO,CAAC;IAChC,sCAAsC;IACtC,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;QAC5C,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;KACzC,EAAE,CAAC;CACL;AAMD,eAAO,MAAM,kBAAkB,EAAE,cAAc,EAW9C,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAMF;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,WAAW,EAAE,EACvB,MAAM,GAAE,WAAkC,GACzC,KAAK,EAAE,CA+CT;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,eAAe,EAAE,WAAW,EAAE,EAC9B,gBAAgB,EAAE,WAAW,EAAE,EAC/B,MAAM,GAAE,WAAkC,GACzC,KAAK,EAAE,CAqDT;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,WAAW,EAAE,EACvB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,EACjE,MAAM,GAAE,WAAkC,GACzC,KAAK,EAAE,CAwCT;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,CA4CpE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,WAAW,EAAE,EACvB,MAAM,GAAE,WAAkC,EAC1C,QAAQ,CAAC,EAAE,WAAW,EAAE,GACvB,WAAW,CA4Db;AAMD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CA6CjE"}
|