@hazeljs/ml 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evaluation/metrics.service.test.js +288 -0
- package/dist/experiments/__tests__/experiment.decorator.test.d.ts +2 -0
- package/dist/experiments/__tests__/experiment.decorator.test.d.ts.map +1 -0
- package/dist/experiments/__tests__/experiment.decorator.test.js +121 -0
- package/dist/experiments/__tests__/experiment.service.test.d.ts +2 -0
- package/dist/experiments/__tests__/experiment.service.test.d.ts.map +1 -0
- package/dist/experiments/__tests__/experiment.service.test.js +460 -0
- package/dist/experiments/experiment.decorator.d.ts +44 -0
- package/dist/experiments/experiment.decorator.d.ts.map +1 -0
- package/dist/experiments/experiment.decorator.js +51 -0
- package/dist/experiments/experiment.service.d.ts +42 -0
- package/dist/experiments/experiment.service.d.ts.map +1 -0
- package/dist/experiments/experiment.service.js +355 -0
- package/dist/experiments/experiment.types.d.ts +60 -0
- package/dist/experiments/experiment.types.d.ts.map +1 -0
- package/dist/experiments/experiment.types.js +5 -0
- package/dist/experiments/index.d.ts +9 -0
- package/dist/experiments/index.d.ts.map +1 -0
- package/dist/experiments/index.js +16 -0
- package/dist/features/__tests__/feature-view.decorator.test.d.ts +2 -0
- package/dist/features/__tests__/feature-view.decorator.test.d.ts.map +1 -0
- package/dist/features/__tests__/feature-view.decorator.test.js +168 -0
- package/dist/features/__tests__/feature.decorator.test.d.ts +2 -0
- package/dist/features/__tests__/feature.decorator.test.d.ts.map +1 -0
- package/dist/features/__tests__/feature.decorator.test.js +167 -0
- package/dist/features/feature-store.service.d.ts +59 -0
- package/dist/features/feature-store.service.d.ts.map +1 -0
- package/dist/features/feature-store.service.js +197 -0
- package/dist/features/feature-view.decorator.d.ts +52 -0
- package/dist/features/feature-view.decorator.d.ts.map +1 -0
- package/dist/features/feature-view.decorator.js +54 -0
- package/dist/features/feature.decorator.d.ts +42 -0
- package/dist/features/feature.decorator.d.ts.map +1 -0
- package/dist/features/feature.decorator.js +49 -0
- package/dist/features/feature.types.d.ts +93 -0
- package/dist/features/feature.types.d.ts.map +1 -0
- package/dist/features/feature.types.js +5 -0
- package/dist/features/index.d.ts +12 -0
- package/dist/features/index.d.ts.map +1 -0
- package/dist/features/index.js +29 -0
- package/dist/features/offline-store.d.ts +40 -0
- package/dist/features/offline-store.d.ts.map +1 -0
- package/dist/features/offline-store.js +215 -0
- package/dist/features/online-store.d.ts +45 -0
- package/dist/features/online-store.d.ts.map +1 -0
- package/dist/features/online-store.js +139 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +26 -1
- package/dist/monitoring/__tests__/drift.service.test.d.ts +2 -0
- package/dist/monitoring/__tests__/drift.service.test.d.ts.map +1 -0
- package/dist/monitoring/__tests__/drift.service.test.js +362 -0
- package/dist/monitoring/__tests__/monitor.service.test.d.ts +2 -0
- package/dist/monitoring/__tests__/monitor.service.test.d.ts.map +1 -0
- package/dist/monitoring/__tests__/monitor.service.test.js +360 -0
- package/dist/monitoring/drift.service.d.ts +68 -0
- package/dist/monitoring/drift.service.d.ts.map +1 -0
- package/dist/monitoring/drift.service.js +360 -0
- package/dist/monitoring/drift.types.d.ts +44 -0
- package/dist/monitoring/drift.types.d.ts.map +1 -0
- package/dist/monitoring/drift.types.js +5 -0
- package/dist/monitoring/index.d.ts +10 -0
- package/dist/monitoring/index.d.ts.map +1 -0
- package/dist/monitoring/index.js +13 -0
- package/dist/monitoring/monitor.service.d.ts +79 -0
- package/dist/monitoring/monitor.service.d.ts.map +1 -0
- package/dist/monitoring/monitor.service.js +192 -0
- package/dist/training/trainer.service.test.js +105 -0
- package/package.json +2 -2
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Drift Service - Statistical drift detection for ML model monitoring
|
|
4
|
+
*/
|
|
5
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
6
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
7
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
8
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
9
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.DriftService = void 0;
|
|
16
|
+
const core_1 = require("@hazeljs/core");
|
|
17
|
+
const core_2 = __importDefault(require("@hazeljs/core"));
|
|
18
|
+
let DriftService = class DriftService {
|
|
19
|
+
constructor() {
|
|
20
|
+
this.referenceDistributions = new Map();
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Set reference distribution for a feature from training data
|
|
24
|
+
*/
|
|
25
|
+
setReferenceDistribution(featureName, values) {
|
|
26
|
+
this.referenceDistributions.set(featureName, [...values]);
|
|
27
|
+
core_2.default.debug(`Set reference distribution for ${featureName} (${values.length} samples)`);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Calculate distribution statistics
|
|
31
|
+
*/
|
|
32
|
+
calculateStats(values) {
|
|
33
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
34
|
+
const count = values.length;
|
|
35
|
+
const min = sorted[0];
|
|
36
|
+
const max = sorted[count - 1];
|
|
37
|
+
const mean = values.reduce((a, b) => a + b, 0) / count;
|
|
38
|
+
// Median
|
|
39
|
+
const mid = Math.floor(count / 2);
|
|
40
|
+
const median = count % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
41
|
+
// Standard deviation
|
|
42
|
+
const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / count;
|
|
43
|
+
const std = Math.sqrt(variance);
|
|
44
|
+
// Histogram (10 bins)
|
|
45
|
+
const binWidth = (max - min) / 10 || 1;
|
|
46
|
+
const histogram = [];
|
|
47
|
+
for (let i = 0; i < 10; i++) {
|
|
48
|
+
const binMin = min + i * binWidth;
|
|
49
|
+
const binMax = binMin + binWidth;
|
|
50
|
+
const binCount = sorted.filter((v) => v >= binMin && v < binMax).length;
|
|
51
|
+
histogram.push({ bin: i, count: binCount });
|
|
52
|
+
}
|
|
53
|
+
// Percentiles
|
|
54
|
+
const percentiles = {};
|
|
55
|
+
for (const p of [5, 25, 50, 75, 95]) {
|
|
56
|
+
const idx = Math.floor((count * p) / 100);
|
|
57
|
+
percentiles[`p${p}`] = sorted[Math.min(idx, count - 1)];
|
|
58
|
+
}
|
|
59
|
+
return { count, min, max, mean, median, std, histogram, percentiles };
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Population Stability Index (PSI) - measures shift between two distributions
|
|
63
|
+
* PSI < 0.1: no significant shift
|
|
64
|
+
* PSI 0.1-0.25: moderate shift
|
|
65
|
+
* PSI > 0.25: significant shift
|
|
66
|
+
*/
|
|
67
|
+
calculatePSI(reference, current, bins = 10) {
|
|
68
|
+
const refStats = this.calculateStats(reference);
|
|
69
|
+
const currStats = this.calculateStats(current);
|
|
70
|
+
const min = Math.min(refStats.min, currStats.min);
|
|
71
|
+
const max = Math.max(refStats.max, currStats.max);
|
|
72
|
+
const binWidth = (max - min) / bins || 1;
|
|
73
|
+
let psi = 0;
|
|
74
|
+
for (let i = 0; i < bins; i++) {
|
|
75
|
+
const binMin = min + i * binWidth;
|
|
76
|
+
const binMax = binMin + binWidth;
|
|
77
|
+
const refCount = reference.filter((v) => v >= binMin && v < binMax).length;
|
|
78
|
+
const currCount = current.filter((v) => v >= binMin && v < binMax).length;
|
|
79
|
+
const refPct = refCount / reference.length;
|
|
80
|
+
const currPct = currCount / current.length;
|
|
81
|
+
// Avoid division by zero
|
|
82
|
+
if (refPct > 0 && currPct > 0) {
|
|
83
|
+
psi += (currPct - refPct) * Math.log(currPct / refPct);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return psi;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Kolmogorov-Smirnov test statistic
|
|
90
|
+
* Measures maximum distance between two cumulative distributions
|
|
91
|
+
* Returns D statistic (0-1) and approximate p-value
|
|
92
|
+
*/
|
|
93
|
+
calculateKS(reference, current) {
|
|
94
|
+
const sortedRef = [...reference].sort((a, b) => a - b);
|
|
95
|
+
const sortedCurr = [...current].sort((a, b) => a - b);
|
|
96
|
+
const allValues = Array.from(new Set([...sortedRef, ...sortedCurr])).sort((a, b) => a - b);
|
|
97
|
+
let maxDiff = 0;
|
|
98
|
+
for (const value of allValues) {
|
|
99
|
+
const refCdf = sortedRef.filter((v) => v <= value).length / sortedRef.length;
|
|
100
|
+
const currCdf = sortedCurr.filter((v) => v <= value).length / sortedCurr.length;
|
|
101
|
+
const diff = Math.abs(refCdf - currCdf);
|
|
102
|
+
if (diff > maxDiff)
|
|
103
|
+
maxDiff = diff;
|
|
104
|
+
}
|
|
105
|
+
// Approximate p-value using Kolmogorov distribution
|
|
106
|
+
const n1 = sortedRef.length;
|
|
107
|
+
const n2 = sortedCurr.length;
|
|
108
|
+
const n = (n1 * n2) / (n1 + n2);
|
|
109
|
+
const lambda = (Math.sqrt(n) + 0.12 + 0.11 / Math.sqrt(n)) * maxDiff;
|
|
110
|
+
// Kolmogorov distribution approximation
|
|
111
|
+
const pValue = Math.max(0, 1 - Math.exp(-2 * lambda * lambda));
|
|
112
|
+
return { d: maxDiff, pValue };
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Jensen-Shannon Divergence - symmetric version of KL divergence
|
|
116
|
+
* Range: 0 (identical) to ln(2) ≈ 0.693 (maximally different)
|
|
117
|
+
*/
|
|
118
|
+
calculateJSD(reference, current, bins = 10) {
|
|
119
|
+
const refStats = this.calculateStats(reference);
|
|
120
|
+
const currStats = this.calculateStats(current);
|
|
121
|
+
const min = Math.min(refStats.min, currStats.min);
|
|
122
|
+
const max = Math.max(refStats.max, currStats.max);
|
|
123
|
+
const binWidth = (max - min) / bins || 1;
|
|
124
|
+
let jsd = 0;
|
|
125
|
+
for (let i = 0; i < bins; i++) {
|
|
126
|
+
const binMin = min + i * binWidth;
|
|
127
|
+
const binMax = binMin + binWidth;
|
|
128
|
+
const refCount = reference.filter((v) => v >= binMin && v < binMax).length;
|
|
129
|
+
const currCount = current.filter((v) => v >= binMin && v < binMax).length;
|
|
130
|
+
const refP = refCount / reference.length;
|
|
131
|
+
const currP = currCount / current.length;
|
|
132
|
+
const avgP = (refP + currP) / 2;
|
|
133
|
+
// KL divergence terms
|
|
134
|
+
if (refP > 0 && avgP > 0) {
|
|
135
|
+
jsd += refP * Math.log(refP / avgP) * 0.5;
|
|
136
|
+
}
|
|
137
|
+
if (currP > 0 && avgP > 0) {
|
|
138
|
+
jsd += currP * Math.log(currP / avgP) * 0.5;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return jsd;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Chi-square test for categorical features
|
|
145
|
+
*/
|
|
146
|
+
calculateChiSquare(reference, current) {
|
|
147
|
+
const allCategories = new Set([...Object.keys(reference), ...Object.keys(current)]);
|
|
148
|
+
const refTotal = Object.values(reference).reduce((a, b) => a + b, 0);
|
|
149
|
+
const currTotal = Object.values(current).reduce((a, b) => a + b, 0);
|
|
150
|
+
let chi2 = 0;
|
|
151
|
+
let df = 0; // degrees of freedom
|
|
152
|
+
for (const category of allCategories) {
|
|
153
|
+
const refCount = reference[category] ?? 0;
|
|
154
|
+
const currCount = current[category] ?? 0;
|
|
155
|
+
const expectedRef = (refCount + currCount) * (refTotal / (refTotal + currTotal));
|
|
156
|
+
const expectedCurr = (refCount + currCount) * (currTotal / (refTotal + currTotal));
|
|
157
|
+
if (expectedRef > 0) {
|
|
158
|
+
chi2 += Math.pow(refCount - expectedRef, 2) / expectedRef;
|
|
159
|
+
df++;
|
|
160
|
+
}
|
|
161
|
+
if (expectedCurr > 0) {
|
|
162
|
+
chi2 += Math.pow(currCount - expectedCurr, 2) / expectedCurr;
|
|
163
|
+
df++;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
df = Math.max(1, df - 1);
|
|
167
|
+
// Approximate p-value using chi-square CDF approximation
|
|
168
|
+
const pValue = this.chiSquarePValue(chi2, df);
|
|
169
|
+
return { chi2, pValue };
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Wasserstein distance (Earth Mover's Distance)
|
|
173
|
+
* Measures how much "work" is needed to transform one distribution into another
|
|
174
|
+
*/
|
|
175
|
+
calculateWasserstein(reference, current) {
|
|
176
|
+
const sortedRef = [...reference].sort((a, b) => a - b);
|
|
177
|
+
const sortedCurr = [...current].sort((a, b) => a - b);
|
|
178
|
+
const n = Math.min(sortedRef.length, sortedCurr.length);
|
|
179
|
+
let distance = 0;
|
|
180
|
+
for (let i = 0; i < n; i++) {
|
|
181
|
+
distance += Math.abs(sortedRef[i] - sortedCurr[i]);
|
|
182
|
+
}
|
|
183
|
+
return distance / n;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Detect drift for numeric features
|
|
187
|
+
*/
|
|
188
|
+
detectDrift(featureName, currentValues, config) {
|
|
189
|
+
const referenceValues = this.referenceDistributions.get(featureName);
|
|
190
|
+
if (!referenceValues) {
|
|
191
|
+
throw new Error(`No reference distribution set for feature: ${featureName}`);
|
|
192
|
+
}
|
|
193
|
+
let score = 0;
|
|
194
|
+
let pValue;
|
|
195
|
+
let driftDetected = false;
|
|
196
|
+
switch (config.method) {
|
|
197
|
+
case 'psi': {
|
|
198
|
+
score = this.calculatePSI(referenceValues, currentValues);
|
|
199
|
+
driftDetected = score > config.threshold;
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
case 'ks': {
|
|
203
|
+
const result = this.calculateKS(referenceValues, currentValues);
|
|
204
|
+
score = result.d;
|
|
205
|
+
pValue = result.pValue;
|
|
206
|
+
driftDetected = result.d > config.threshold || result.pValue < 0.05;
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
case 'jsd': {
|
|
210
|
+
score = this.calculateJSD(referenceValues, currentValues);
|
|
211
|
+
driftDetected = score > config.threshold;
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
case 'wasserstein': {
|
|
215
|
+
score = this.calculateWasserstein(referenceValues, currentValues);
|
|
216
|
+
// Normalize by standard deviation
|
|
217
|
+
const refStats = this.calculateStats(referenceValues);
|
|
218
|
+
score = refStats.std > 0 ? score / refStats.std : score;
|
|
219
|
+
driftDetected = score > config.threshold;
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
default:
|
|
223
|
+
throw new Error(`Unsupported drift detection method: ${config.method}`);
|
|
224
|
+
}
|
|
225
|
+
const message = driftDetected
|
|
226
|
+
? `Drift detected in ${featureName}: ${config.method}=${score.toFixed(4)} exceeds threshold ${config.threshold}`
|
|
227
|
+
: `No drift detected in ${featureName}: ${config.method}=${score.toFixed(4)}`;
|
|
228
|
+
return {
|
|
229
|
+
feature: featureName,
|
|
230
|
+
driftDetected,
|
|
231
|
+
score,
|
|
232
|
+
threshold: config.threshold,
|
|
233
|
+
method: config.method,
|
|
234
|
+
pValue,
|
|
235
|
+
message,
|
|
236
|
+
timestamp: new Date(),
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Detect drift for categorical features
|
|
241
|
+
*/
|
|
242
|
+
detectCategoricalDrift(featureName, currentValues, config) {
|
|
243
|
+
// Count frequencies
|
|
244
|
+
const referenceValues = this.referenceDistributions.get(featureName);
|
|
245
|
+
if (!referenceValues) {
|
|
246
|
+
throw new Error(`No reference distribution set for feature: ${featureName}`);
|
|
247
|
+
}
|
|
248
|
+
const refCounts = this.countCategories(referenceValues);
|
|
249
|
+
const currCounts = this.countCategories(currentValues);
|
|
250
|
+
const { chi2, pValue } = this.calculateChiSquare(refCounts, currCounts);
|
|
251
|
+
// Normalize chi2 score
|
|
252
|
+
const score = chi2 / Math.max(1, Object.keys(refCounts).length);
|
|
253
|
+
const driftDetected = pValue < 0.05 || score > config.threshold;
|
|
254
|
+
return {
|
|
255
|
+
feature: featureName,
|
|
256
|
+
driftDetected,
|
|
257
|
+
score,
|
|
258
|
+
threshold: config.threshold,
|
|
259
|
+
method: 'chi2',
|
|
260
|
+
pValue,
|
|
261
|
+
message: driftDetected
|
|
262
|
+
? `Drift detected in ${featureName}: chi2=${score.toFixed(4)}, p=${pValue?.toFixed(4)}`
|
|
263
|
+
: `No drift detected in ${featureName}: chi2=${score.toFixed(4)}`,
|
|
264
|
+
timestamp: new Date(),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Run full drift detection report on multiple features
|
|
269
|
+
*/
|
|
270
|
+
detectDriftReport(features, config) {
|
|
271
|
+
const results = [];
|
|
272
|
+
for (const [name, values] of Object.entries(features)) {
|
|
273
|
+
try {
|
|
274
|
+
const result = this.detectDrift(name, values, config);
|
|
275
|
+
results.push(result);
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
core_2.default.warn(`Failed to detect drift for ${name}:`, error);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const driftedFeatures = results.filter((r) => r.driftDetected).length;
|
|
282
|
+
const totalFeatures = results.length;
|
|
283
|
+
const driftPercentage = totalFeatures > 0 ? (driftedFeatures / totalFeatures) * 100 : 0;
|
|
284
|
+
return {
|
|
285
|
+
timestamp: new Date(),
|
|
286
|
+
totalFeatures,
|
|
287
|
+
driftedFeatures,
|
|
288
|
+
driftPercentage,
|
|
289
|
+
results,
|
|
290
|
+
overallDrift: driftPercentage > 25, // Overall drift if >25% of features drifted
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Detect prediction drift (monitor model output distribution)
|
|
295
|
+
*/
|
|
296
|
+
detectPredictionDrift(referencePredictions, currentPredictions) {
|
|
297
|
+
// For numeric predictions (regression)
|
|
298
|
+
if (typeof referencePredictions[0] === 'number') {
|
|
299
|
+
this.setReferenceDistribution('__prediction__', referencePredictions);
|
|
300
|
+
return this.detectDrift('__prediction__', currentPredictions, {
|
|
301
|
+
method: 'ks',
|
|
302
|
+
threshold: 0.1,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
// For categorical predictions (classification)
|
|
306
|
+
const refCounts = this.countCategories(referencePredictions);
|
|
307
|
+
const currCounts = this.countCategories(currentPredictions);
|
|
308
|
+
const { chi2, pValue } = this.calculateChiSquare(refCounts, currCounts);
|
|
309
|
+
const score = chi2 / Math.max(1, Object.keys(refCounts).length);
|
|
310
|
+
const driftDetected = pValue < 0.05;
|
|
311
|
+
return {
|
|
312
|
+
feature: 'prediction',
|
|
313
|
+
driftDetected,
|
|
314
|
+
score,
|
|
315
|
+
threshold: 0.05,
|
|
316
|
+
method: 'chi2',
|
|
317
|
+
pValue,
|
|
318
|
+
message: driftDetected
|
|
319
|
+
? `Prediction drift detected: chi2=${score.toFixed(4)}, p=${pValue?.toFixed(4)}`
|
|
320
|
+
: `No prediction drift detected: chi2=${score.toFixed(4)}`,
|
|
321
|
+
timestamp: new Date(),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
|
325
|
+
countCategories(values) {
|
|
326
|
+
const counts = {};
|
|
327
|
+
for (const value of values) {
|
|
328
|
+
counts[value] = (counts[value] ?? 0) + 1;
|
|
329
|
+
}
|
|
330
|
+
return counts;
|
|
331
|
+
}
|
|
332
|
+
chiSquarePValue(chi2, df) {
|
|
333
|
+
// Wilson-Hilferty approximation for chi-square CDF
|
|
334
|
+
if (chi2 <= 0)
|
|
335
|
+
return 1;
|
|
336
|
+
if (df <= 0)
|
|
337
|
+
return 1;
|
|
338
|
+
const z = Math.sqrt(2 * chi2) - Math.sqrt(2 * df - 1);
|
|
339
|
+
// Standard normal CDF approximation
|
|
340
|
+
return 1 - this.normalCDF(z);
|
|
341
|
+
}
|
|
342
|
+
normalCDF(x) {
|
|
343
|
+
// Abramowitz and Stegun approximation
|
|
344
|
+
const a1 = 0.254829592;
|
|
345
|
+
const a2 = -0.284496736;
|
|
346
|
+
const a3 = 1.421413741;
|
|
347
|
+
const a4 = -1.453152027;
|
|
348
|
+
const a5 = 1.061405429;
|
|
349
|
+
const p = 0.3275911;
|
|
350
|
+
const sign = x < 0 ? -1 : 1;
|
|
351
|
+
const absX = Math.abs(x) / Math.sqrt(2);
|
|
352
|
+
const t = 1 / (1 + p * absX);
|
|
353
|
+
const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX);
|
|
354
|
+
return 0.5 * (1 + sign * y);
|
|
355
|
+
}
|
|
356
|
+
};
|
|
357
|
+
exports.DriftService = DriftService;
|
|
358
|
+
exports.DriftService = DriftService = __decorate([
|
|
359
|
+
(0, core_1.Service)()
|
|
360
|
+
], DriftService);
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @hazeljs/ml - Drift Detection Types
|
|
3
|
+
*/
|
|
4
|
+
export type DriftType = 'data' | 'prediction' | 'concept';
|
|
5
|
+
export interface DriftConfig {
|
|
6
|
+
type: DriftType;
|
|
7
|
+
features: string[];
|
|
8
|
+
method: 'psi' | 'ks' | 'jsd' | 'chi2' | 'wasserstein';
|
|
9
|
+
threshold: number;
|
|
10
|
+
windowSize?: number;
|
|
11
|
+
referenceDistribution?: Record<string, number[]>;
|
|
12
|
+
}
|
|
13
|
+
export interface DriftResult {
|
|
14
|
+
feature: string;
|
|
15
|
+
driftDetected: boolean;
|
|
16
|
+
score: number;
|
|
17
|
+
threshold: number;
|
|
18
|
+
method: DriftConfig['method'];
|
|
19
|
+
pValue?: number;
|
|
20
|
+
message: string;
|
|
21
|
+
timestamp: Date;
|
|
22
|
+
}
|
|
23
|
+
export interface DriftReport {
|
|
24
|
+
timestamp: Date;
|
|
25
|
+
totalFeatures: number;
|
|
26
|
+
driftedFeatures: number;
|
|
27
|
+
driftPercentage: number;
|
|
28
|
+
results: DriftResult[];
|
|
29
|
+
overallDrift: boolean;
|
|
30
|
+
}
|
|
31
|
+
export interface DistributionStats {
|
|
32
|
+
count: number;
|
|
33
|
+
min: number;
|
|
34
|
+
max: number;
|
|
35
|
+
mean: number;
|
|
36
|
+
median: number;
|
|
37
|
+
std: number;
|
|
38
|
+
histogram: Array<{
|
|
39
|
+
bin: number;
|
|
40
|
+
count: number;
|
|
41
|
+
}>;
|
|
42
|
+
percentiles: Record<string, number>;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=drift.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"drift.types.d.ts","sourceRoot":"","sources":["../../src/monitoring/drift.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;AAE1D,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,GAAG,aAAa,CAAC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CAClD;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,IAAI,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @hazeljs/ml - Monitoring & Drift Detection
|
|
3
|
+
*
|
|
4
|
+
* Export all monitoring components
|
|
5
|
+
*/
|
|
6
|
+
export type { DriftType, DriftConfig, DriftResult, DriftReport, DistributionStats, } from './drift.types';
|
|
7
|
+
export type { MonitorConfig, MonitorAlert, AlertHandler } from './monitor.service';
|
|
8
|
+
export { DriftService } from './drift.service';
|
|
9
|
+
export { MonitorService } from './monitor.service';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/monitoring/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,SAAS,EACT,WAAW,EACX,WAAW,EACX,WAAW,EACX,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGnF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @hazeljs/ml - Monitoring & Drift Detection
|
|
4
|
+
*
|
|
5
|
+
* Export all monitoring components
|
|
6
|
+
*/
|
|
7
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
+
exports.MonitorService = exports.DriftService = void 0;
|
|
9
|
+
// Services
|
|
10
|
+
var drift_service_1 = require("./drift.service");
|
|
11
|
+
Object.defineProperty(exports, "DriftService", { enumerable: true, get: function () { return drift_service_1.DriftService; } });
|
|
12
|
+
var monitor_service_1 = require("./monitor.service");
|
|
13
|
+
Object.defineProperty(exports, "MonitorService", { enumerable: true, get: function () { return monitor_service_1.MonitorService; } });
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Monitor Service - Ongoing model monitoring and alerting
|
|
3
|
+
*/
|
|
4
|
+
import type { DriftResult, DriftConfig } from './drift.types';
|
|
5
|
+
import { DriftService } from './drift.service';
|
|
6
|
+
export interface MonitorConfig {
|
|
7
|
+
modelName: string;
|
|
8
|
+
modelVersion?: string;
|
|
9
|
+
featureDrift?: Omit<DriftConfig, 'features' | 'type'>;
|
|
10
|
+
predictionDrift?: boolean;
|
|
11
|
+
accuracyMonitor?: {
|
|
12
|
+
threshold: number;
|
|
13
|
+
windowSize: number;
|
|
14
|
+
};
|
|
15
|
+
alertWebhook?: string;
|
|
16
|
+
checkIntervalMinutes?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface MonitorAlert {
|
|
19
|
+
timestamp: Date;
|
|
20
|
+
modelName: string;
|
|
21
|
+
modelVersion?: string;
|
|
22
|
+
alertType: 'drift' | 'accuracy' | 'latency' | 'error_rate';
|
|
23
|
+
severity: 'warning' | 'critical';
|
|
24
|
+
message: string;
|
|
25
|
+
details: Record<string, unknown>;
|
|
26
|
+
}
|
|
27
|
+
export type AlertHandler = (alert: MonitorAlert) => void | Promise<void>;
|
|
28
|
+
export declare class MonitorService {
|
|
29
|
+
private driftService;
|
|
30
|
+
private monitors;
|
|
31
|
+
private alertHandlers;
|
|
32
|
+
private checkIntervals;
|
|
33
|
+
private accuracyHistory;
|
|
34
|
+
constructor(driftService: DriftService);
|
|
35
|
+
/**
|
|
36
|
+
* Register a model for monitoring
|
|
37
|
+
*/
|
|
38
|
+
registerModel(config: MonitorConfig): void;
|
|
39
|
+
/**
|
|
40
|
+
* Unregister a model from monitoring
|
|
41
|
+
*/
|
|
42
|
+
unregisterModel(modelName: string, modelVersion?: string): void;
|
|
43
|
+
/**
|
|
44
|
+
* Add an alert handler
|
|
45
|
+
*/
|
|
46
|
+
onAlert(handler: AlertHandler): void;
|
|
47
|
+
/**
|
|
48
|
+
* Remove an alert handler
|
|
49
|
+
*/
|
|
50
|
+
offAlert(handler: AlertHandler): void;
|
|
51
|
+
/**
|
|
52
|
+
* Record prediction for drift monitoring
|
|
53
|
+
*/
|
|
54
|
+
recordPrediction(modelName: string, features: Record<string, number>, prediction: number | string): void;
|
|
55
|
+
/**
|
|
56
|
+
* Record accuracy metric for accuracy monitoring
|
|
57
|
+
*/
|
|
58
|
+
recordAccuracy(modelName: string, accuracy: number, modelVersion?: string): void;
|
|
59
|
+
/**
|
|
60
|
+
* Check a model for drift and other issues
|
|
61
|
+
*/
|
|
62
|
+
checkModel(modelName: string, modelVersion?: string): Promise<DriftResult[]>;
|
|
63
|
+
/**
|
|
64
|
+
* Get monitoring status for all registered models
|
|
65
|
+
*/
|
|
66
|
+
getStatus(): Array<{
|
|
67
|
+
modelName: string;
|
|
68
|
+
modelVersion?: string;
|
|
69
|
+
isActive: boolean;
|
|
70
|
+
checkInterval?: number;
|
|
71
|
+
}>;
|
|
72
|
+
/**
|
|
73
|
+
* Stop all monitoring
|
|
74
|
+
*/
|
|
75
|
+
stop(): void;
|
|
76
|
+
private getMonitorKey;
|
|
77
|
+
private emitAlert;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=monitor.service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"monitor.service.d.ts","sourceRoot":"","sources":["../../src/monitoring/monitor.service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,GAAG,MAAM,CAAC,CAAC;IACtD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,eAAe,CAAC,EAAE;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC;IAC3D,QAAQ,EAAE,SAAS,GAAG,UAAU,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAEzE,qBACa,cAAc;IACzB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAyC;IACzD,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAA0C;IAChE,OAAO,CAAC,eAAe,CAAwE;gBAEnF,YAAY,EAAE,YAAY;IAItC;;OAEG;IACH,aAAa,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAsB1C;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;IAa/D;;OAEG;IACH,OAAO,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAIpC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAOrC;;OAEG;IACH,gBAAgB,CACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,UAAU,EAAE,MAAM,GAAG,MAAM,GAC1B,IAAI;IAMP;;OAEG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;IA2BhF;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA0ClF;;OAEG;IACH,SAAS,IAAI,KAAK,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,EAAE,OAAO,CAAC;QAClB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IASF;;OAEG;IACH,IAAI,IAAI,IAAI;IAQZ,OAAO,CAAC,aAAa;YAIP,SAAS;CASxB"}
|