@hazeljs/ml 0.2.4 → 0.3.1

package/dist/monitoring/__tests__/monitor.service.test.js

@@ -0,0 +1,360 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const monitor_service_1 = require("../monitor.service");
+const drift_service_1 = require("../drift.service");
+describe('MonitorService', () => {
+    let service;
+    let driftService;
+    beforeEach(() => {
+        driftService = new drift_service_1.DriftService();
+        service = new monitor_service_1.MonitorService(driftService);
+    });
+    afterEach(() => {
+        service.stop();
+    });
+    describe('registerModel', () => {
+        it('should register a model for monitoring', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                modelVersion: '1.0.0',
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(1);
+            expect(status[0].modelName).toBe('test-model');
+            expect(status[0].modelVersion).toBe('1.0.0');
+        });
+        it('should register model without version', () => {
+            service.registerModel({
+                modelName: 'test-model',
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(1);
+            expect(status[0].modelName).toBe('test-model');
+            expect(status[0].modelVersion).toBeUndefined();
+        });
+        it('should set up periodic checks when interval specified', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                checkIntervalMinutes: 5,
+            });
+            const status = service.getStatus();
+            expect(status[0].isActive).toBe(true);
+            expect(status[0].checkInterval).toBe(5);
+        });
+        it('should not set up periodic checks when interval is 0', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                checkIntervalMinutes: 0,
+            });
+            const status = service.getStatus();
+            expect(status[0].isActive).toBe(false);
+        });
+        it('should replace old interval when re-registering', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                checkIntervalMinutes: 5,
+            });
+            service.registerModel({
+                modelName: 'test-model',
+                checkIntervalMinutes: 10,
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(1);
+            expect(status[0].checkInterval).toBe(10);
+        });
+        it('should register with feature drift config', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                featureDrift: {
+                    method: 'psi',
+                    threshold: 0.25,
+                },
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(1);
+        });
+        it('should register with accuracy monitor', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.8,
+                    windowSize: 10,
+                },
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(1);
+        });
+    });
+    describe('unregisterModel', () => {
+        it('should unregister a model', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                modelVersion: '1.0.0',
+            });
+            service.unregisterModel('test-model', '1.0.0');
+            const status = service.getStatus();
+            expect(status).toHaveLength(0);
+        });
+        it('should clear interval when unregistering', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                checkIntervalMinutes: 5,
+            });
+            service.unregisterModel('test-model');
+            const status = service.getStatus();
+            expect(status).toHaveLength(0);
+        });
+        it('should handle unregistering non-existent model', () => {
+            expect(() => {
+                service.unregisterModel('non-existent');
+            }).not.toThrow();
+        });
+    });
+    describe('onAlert', () => {
+        it('should add alert handler', () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 2,
+                },
+            });
+            service.recordAccuracy('test-model', 0.5);
+            service.recordAccuracy('test-model', 0.5);
+            expect(handler).toHaveBeenCalled();
+        });
+        it('should support multiple alert handlers', async () => {
+            const handler1 = jest.fn();
+            const handler2 = jest.fn();
+            service.onAlert(handler1);
+            service.onAlert(handler2);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 2,
+                },
+            });
+            service.recordAccuracy('test-model', 0.5);
+            service.recordAccuracy('test-model', 0.5);
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            expect(handler1).toHaveBeenCalled();
+            expect(handler2).toHaveBeenCalled();
+        });
+    });
+    describe('offAlert', () => {
+        it('should remove alert handler', () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            service.offAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 1,
+                },
+            });
+            service.recordAccuracy('test-model', 0.5);
+            expect(handler).not.toHaveBeenCalled();
+        });
+        it('should handle removing non-existent handler', () => {
+            const handler = jest.fn();
+            expect(() => {
+                service.offAlert(handler);
+            }).not.toThrow();
+        });
+    });
+    describe('recordPrediction', () => {
+        it('should record prediction', () => {
+            expect(() => {
+                service.recordPrediction('test-model', { feature1: 1.0, feature2: 2.0 }, 'positive');
+            }).not.toThrow();
+        });
+        it('should record numeric prediction', () => {
+            expect(() => {
+                service.recordPrediction('test-model', { feature1: 1.0 }, 0.95);
+            }).not.toThrow();
+        });
+    });
+    describe('recordAccuracy', () => {
+        it('should record accuracy', () => {
+            service.registerModel({
+                modelName: 'test-model',
+            });
+            expect(() => {
+                service.recordAccuracy('test-model', 0.95);
+            }).not.toThrow();
+        });
+        it('should record accuracy with version', () => {
+            service.registerModel({
+                modelName: 'test-model',
+                modelVersion: '1.0.0',
+            });
+            expect(() => {
+                service.recordAccuracy('test-model', 0.95, '1.0.0');
+            }).not.toThrow();
+        });
+        it('should trigger alert when accuracy drops below threshold', () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 2,
+                },
+            });
+            service.recordAccuracy('test-model', 0.85);
+            service.recordAccuracy('test-model', 0.85);
+            expect(handler).toHaveBeenCalled();
+            const alert = handler.mock.calls[0][0];
+            expect(alert.alertType).toBe('accuracy');
+            expect(alert.severity).toBe('critical');
+            expect(alert.modelName).toBe('test-model');
+        });
+        it('should not trigger alert when accuracy is above threshold', () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 2,
+                },
+            });
+            service.recordAccuracy('test-model', 0.95);
+            service.recordAccuracy('test-model', 0.95);
+            expect(handler).not.toHaveBeenCalled();
+        });
+        it('should use window size for accuracy calculation', () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 2,
+                },
+            });
+            service.recordAccuracy('test-model', 0.95);
+            service.recordAccuracy('test-model', 0.95);
+            service.recordAccuracy('test-model', 0.85);
+            service.recordAccuracy('test-model', 0.85);
+            expect(handler).toHaveBeenCalled();
+        });
+    });
+    describe('checkModel', () => {
+        it('should throw error for unregistered model', async () => {
+            await expect(service.checkModel('non-existent')).rejects.toThrow('No monitor registered');
+        });
+        it('should check model without drift config', async () => {
+            service.registerModel({
+                modelName: 'test-model',
+            });
+            const results = await service.checkModel('test-model');
+            expect(results).toEqual([]);
+        });
+        it('should check model with feature drift config', async () => {
+            driftService.setReferenceDistribution('feature1', [1, 2, 3, 4, 5]);
+            service.registerModel({
+                modelName: 'test-model',
+                featureDrift: {
+                    method: 'psi',
+                    threshold: 0.25,
+                    windowSize: 100,
+                },
+            });
+            const results = await service.checkModel('test-model');
+            expect(Array.isArray(results)).toBe(true);
+        });
+        it('should emit alert when drift detected', async () => {
+            const handler = jest.fn();
+            service.onAlert(handler);
+            driftService.setReferenceDistribution('feature1', [1, 2, 3, 4, 5]);
+            service.registerModel({
+                modelName: 'test-model',
+                featureDrift: {
+                    method: 'psi',
+                    threshold: 0.01,
+                    windowSize: 100,
+                },
+            });
+            await service.checkModel('test-model');
+            // Alert may or may not be called depending on drift detection
+            expect(handler).toHaveBeenCalledTimes(0);
+        });
+    });
+    describe('getStatus', () => {
+        it('should return empty array when no models registered', () => {
+            expect(service.getStatus()).toEqual([]);
+        });
+        it('should return status for all registered models', () => {
+            service.registerModel({
+                modelName: 'model1',
+                modelVersion: '1.0.0',
+            });
+            service.registerModel({
+                modelName: 'model2',
+                checkIntervalMinutes: 5,
+            });
+            const status = service.getStatus();
+            expect(status).toHaveLength(2);
+            expect(status[0].modelName).toBe('model1');
+            expect(status[1].modelName).toBe('model2');
+            expect(status[1].isActive).toBe(true);
+        });
+    });
+    describe('stop', () => {
+        it('should stop all monitoring', () => {
+            service.registerModel({
+                modelName: 'model1',
+                checkIntervalMinutes: 5,
+            });
+            service.registerModel({
+                modelName: 'model2',
+                checkIntervalMinutes: 10,
+            });
+            service.stop();
+            const status = service.getStatus();
+            expect(status[0].isActive).toBe(false);
+            expect(status[1].isActive).toBe(false);
+        });
+        it('should handle stopping when no models registered', () => {
+            expect(() => {
+                service.stop();
+            }).not.toThrow();
+        });
+    });
+    describe('alert handling', () => {
+        it('should handle async alert handlers', async () => {
+            const handler = jest.fn().mockResolvedValue(undefined);
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 1,
+                },
+            });
+            service.recordAccuracy('test-model', 0.5);
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            expect(handler).toHaveBeenCalled();
+        });
+        it('should handle alert handler errors gracefully', async () => {
+            const handler = jest.fn().mockRejectedValue(new Error('Handler error'));
+            service.onAlert(handler);
+            service.registerModel({
+                modelName: 'test-model',
+                accuracyMonitor: {
+                    threshold: 0.9,
+                    windowSize: 1,
+                },
+            });
+            expect(() => {
+                service.recordAccuracy('test-model', 0.5);
+            }).not.toThrow();
+        });
+    });
+});

package/dist/monitoring/drift.service.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Drift Service - Statistical drift detection for ML model monitoring
+ */
+import type { DriftConfig, DriftResult, DriftReport, DistributionStats } from './drift.types';
+export declare class DriftService {
+    private referenceDistributions;
+    /**
+     * Set reference distribution for a feature from training data
+     */
+    setReferenceDistribution(featureName: string, values: number[]): void;
+    /**
+     * Calculate distribution statistics
+     */
+    calculateStats(values: number[]): DistributionStats;
+    /**
+     * Population Stability Index (PSI) - measures shift between two distributions
+     * PSI < 0.1: no significant shift
+     * PSI 0.1-0.25: moderate shift
+     * PSI > 0.25: significant shift
+     */
+    calculatePSI(reference: number[], current: number[], bins?: number): number;
+    /**
+     * Kolmogorov-Smirnov test statistic
+     * Measures maximum distance between two cumulative distributions
+     * Returns D statistic (0-1) and approximate p-value
+     */
+    calculateKS(reference: number[], current: number[]): {
+        d: number;
+        pValue: number;
+    };
+    /**
+     * Jensen-Shannon Divergence - symmetric version of KL divergence
+     * Range: 0 (identical) to ln(2) ≈ 0.693 (maximally different)
+     */
+    calculateJSD(reference: number[], current: number[], bins?: number): number;
+    /**
+     * Chi-square test for categorical features
+     */
+    calculateChiSquare(reference: Record<string, number>, current: Record<string, number>): {
+        chi2: number;
+        pValue: number;
+    };
+    /**
+     * Wasserstein distance (Earth Mover's Distance)
+     * Measures how much "work" is needed to transform one distribution into another
+     */
+    calculateWasserstein(reference: number[], current: number[]): number;
+    /**
+     * Detect drift for numeric features
+     */
+    detectDrift(featureName: string, currentValues: number[], config: Omit<DriftConfig, 'features' | 'type'>): DriftResult;
+    /**
+     * Detect drift for categorical features
+     */
+    detectCategoricalDrift(featureName: string, currentValues: string[], config: Omit<DriftConfig, 'features' | 'type' | 'method'>): DriftResult;
+    /**
+     * Run full drift detection report on multiple features
+     */
+    detectDriftReport(features: Record<string, number[]>, config: Pick<DriftConfig, 'method' | 'threshold' | 'windowSize'>): DriftReport;
+    /**
+     * Detect prediction drift (monitor model output distribution)
+     */
+    detectPredictionDrift(referencePredictions: number[] | string[], currentPredictions: number[] | string[]): DriftResult;
+    private countCategories;
+    private chiSquarePValue;
+    private normalCDF;
+}
+//# sourceMappingURL=drift.service.d.ts.map

package/dist/monitoring/drift.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drift.service.d.ts","sourceRoot":"","sources":["../../src/monitoring/drift.service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAE9F,qBACa,YAAY;IACvB,OAAO,CAAC,sBAAsB,CAAoC;IAElE;;OAEG;IACH,wBAAwB,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAKrE;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB;IAmCnD;;;;;OAKG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,SAAK,GAAG,MAAM;IA6BvE;;;;OAIG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IA0BlF;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,SAAK,GAAG,MAAM;IAiCvE;;OAEG;IACH,kBAAkB,CAChB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IAkCnC;;;OAGG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM;IAcpE;;OAEG;IACH,WAAW,CACT,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,EAAE,EACvB,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,GAAG,MAAM,CAAC,GAC7C,WAAW;IAwDd;;OAEG;IACH,sBAAsB,CACpB,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,MAAM,EAAE,EACvB,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC,GACxD,WAAW;IA8Bd;;OAEG;IACH,iBAAiB,CACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAClC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,GAAG,WAAW,GAAG,YAAY,CAAC,GAC/D,WAAW;IA0Bd;;OAEG;IACH,qBAAqB,CACnB,oBAAoB,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EACzC,kBAAkB,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GACtC,WAAW;IAkCd,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,SAAS;CAiBlB"}