@hazeljs/ml 0.2.4 → 0.3.0

package/dist/evaluation/metrics.service.test.js

@@ -249,5 +249,293 @@ describe('MetricsService', () => {
             const result = await svc.evaluate('eval-model', [{ text: 'good', label: 'positive' }]);
             expect(result.version).toBeDefined();
         });
+        it('evaluate with precision metric only', async () => {
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'eval-model', version: '1.0.0', framework: 'custom' },
+                instance: new EvalModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('eval-model', [
+                { text: 'good', label: 'positive' },
+                { text: 'bad', label: 'negative' },
+            ], { metrics: ['precision'] });
+            expect(result.metrics.precision).toBeDefined();
+            expect(result.metrics.accuracy).toBeUndefined();
+        });
+        it('evaluate with recall metric only', async () => {
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'eval-model', version: '1.0.0', framework: 'custom' },
+                instance: new EvalModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('eval-model', [
+                { text: 'good', label: 'positive' },
+                { text: 'bad', label: 'negative' },
+            ], { metrics: ['recall'] });
+            expect(result.metrics.recall).toBeDefined();
+            expect(result.metrics.accuracy).toBeUndefined();
+        });
+        it('evaluate with f1 metric only', async () => {
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'eval-model', version: '1.0.0', framework: 'custom' },
+                instance: new EvalModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('eval-model', [
+                { text: 'good', label: 'positive' },
+                { text: 'bad', label: 'negative' },
+            ], { metrics: ['f1'] });
+            expect(result.metrics.f1Score).toBeDefined();
+            expect(result.metrics.accuracy).toBeUndefined();
+        });
+    });
+    describe('getMetrics edge cases', () => {
+        it('should return undefined for non-existent model', () => {
+            expect(service.getMetrics('non-existent')).toBeUndefined();
+        });
+        it('should return undefined for non-existent version', () => {
+            service.recordEvaluation({
+                modelName: 'model',
+                version: '1.0.0',
+                metrics: { accuracy: 0.9 },
+                evaluatedAt: new Date(),
+            });
+            expect(service.getMetrics('model', '2.0.0')).toBeUndefined();
+        });
+    });
+    describe('getHistory edge cases', () => {
+        it('should return empty array for non-existent model', () => {
+            expect(service.getHistory('non-existent')).toEqual([]);
+        });
+    });
+    describe('compareVersions edge cases', () => {
+        it('should return undefined winner when both versions not found', () => {
+            const { a, b, winner } = service.compareVersions('model', 'v1', 'v2');
+            expect(a).toBeUndefined();
+            expect(b).toBeUndefined();
+            expect(winner).toBeUndefined();
+        });
+        it('should return undefined winner when only one version found', () => {
+            service.recordEvaluation({
+                modelName: 'model',
+                version: 'v1',
+                metrics: { accuracy: 0.9 },
+                evaluatedAt: new Date(),
+            });
+            const { a, b, winner } = service.compareVersions('model', 'v1', 'v2');
+            expect(a).toBeDefined();
+            expect(b).toBeUndefined();
+            expect(winner).toBeUndefined();
+        });
+        it('should handle tie in accuracy', () => {
+            service.recordEvaluation({
+                modelName: 'model',
+                version: 'v1',
+                metrics: { accuracy: 0.9 },
+                evaluatedAt: new Date(),
+            });
+            service.recordEvaluation({
+                modelName: 'model',
+                version: 'v2',
+                metrics: { accuracy: 0.9 },
+                evaluatedAt: new Date(),
+            });
+            const { winner } = service.compareVersions('model', 'v1', 'v2');
+            expect(winner).toBe('v1'); // First one wins in tie
+        });
+    });
+    describe('evaluate with all metrics', () => {
+        let FullModel = class FullModel {
+            train() { }
+            async predict(input) {
+                return { label: input.text.includes('good') ? 'positive' : 'negative' };
+            }
+        };
+        __decorate([
+            (0, decorators_1.Train)(),
+            __metadata("design:type", Function),
+            __metadata("design:paramtypes", []),
+            __metadata("design:returntype", void 0)
+        ], FullModel.prototype, "train", null);
+        __decorate([
+            (0, decorators_1.Predict)(),
+            __metadata("design:type", Function),
+            __metadata("design:paramtypes", [Object]),
+            __metadata("design:returntype", Promise)
+        ], FullModel.prototype, "predict", null);
+        FullModel = __decorate([
+            (0, decorators_1.Model)({ name: 'full-model', version: '1.0.0', framework: 'custom' })
+        ], FullModel);
+        it('should compute all metrics when requested', async () => {
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'full-model', version: '1.0.0', framework: 'custom' },
+                instance: new FullModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('full-model', [
+                { text: 'good', label: 'positive' },
+                { text: 'bad', label: 'negative' },
+                { text: 'good day', label: 'positive' },
+                { text: 'bad day', label: 'negative' },
+            ], { metrics: ['accuracy', 'precision', 'recall', 'f1'] });
+            expect(result.metrics.accuracy).toBeDefined();
+            expect(result.metrics.precision).toBeDefined();
+            expect(result.metrics.recall).toBeDefined();
+            expect(result.metrics.f1Score).toBeDefined();
+        });
+        it('should handle mixed correct and incorrect predictions', async () => {
+            let MixedModel = class MixedModel {
+                train() { }
+                async predict(input) {
+                    return { prediction: input.value > 5 ? 'high' : 'low' };
+                }
+            };
+            __decorate([
+                (0, decorators_1.Train)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", []),
+                __metadata("design:returntype", void 0)
+            ], MixedModel.prototype, "train", null);
+            __decorate([
+                (0, decorators_1.Predict)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", [Object]),
+                __metadata("design:returntype", Promise)
+            ], MixedModel.prototype, "predict", null);
+            MixedModel = __decorate([
+                (0, decorators_1.Model)({ name: 'mixed-model', version: '1.0.0', framework: 'custom' })
+            ], MixedModel);
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'mixed-model', version: '1.0.0', framework: 'custom' },
+                instance: new MixedModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('mixed-model', [
+                { value: 10, label: 'high' },
+                { value: 2, label: 'low' },
+                { value: 8, label: 'low' }, // Wrong
+                { value: 3, label: 'high' }, // Wrong
+            ], { labelKey: 'label', predictionKey: 'prediction' });
+            expect(result.metrics.accuracy).toBe(0.5);
+        });
+        it('should calculate precision correctly with false positives', async () => {
+            let PrecisionModel = class PrecisionModel {
+                train() { }
+                async predict() {
+                    return { result: 'positive' }; // Always predicts positive
+                }
+            };
+            __decorate([
+                (0, decorators_1.Train)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", []),
+                __metadata("design:returntype", void 0)
+            ], PrecisionModel.prototype, "train", null);
+            __decorate([
+                (0, decorators_1.Predict)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", []),
+                __metadata("design:returntype", Promise)
+            ], PrecisionModel.prototype, "predict", null);
+            PrecisionModel = __decorate([
+                (0, decorators_1.Model)({ name: 'precision-model', version: '1.0.0', framework: 'custom' })
+            ], PrecisionModel);
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'precision-model', version: '1.0.0', framework: 'custom' },
+                instance: new PrecisionModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('precision-model', [
+                { text: 'a', label: 'positive' },
+                { text: 'b', label: 'negative' },
+                { text: 'c', label: 'negative' },
+            ], { labelKey: 'label', predictionKey: 'result', metrics: ['precision'] });
+            expect(result.metrics.precision).toBeLessThan(1);
+        });
+        it('should calculate recall correctly with false negatives', async () => {
+            let RecallModel = class RecallModel {
+                train() { }
+                async predict() {
+                    return { result: 'negative' }; // Always predicts negative
+                }
+            };
+            __decorate([
+                (0, decorators_1.Train)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", []),
+                __metadata("design:returntype", void 0)
+            ], RecallModel.prototype, "train", null);
+            __decorate([
+                (0, decorators_1.Predict)(),
+                __metadata("design:type", Function),
+                __metadata("design:paramtypes", []),
+                __metadata("design:returntype", Promise)
+            ], RecallModel.prototype, "predict", null);
+            RecallModel = __decorate([
+                (0, decorators_1.Model)({ name: 'recall-model', version: '1.0.0', framework: 'custom' })
+            ], RecallModel);
+            const registry = new model_registry_1.ModelRegistry();
+            registry.register({
+                metadata: { name: 'recall-model', version: '1.0.0', framework: 'custom' },
+                instance: new RecallModel(),
+                predictMethod: 'predict',
+            });
+            const predictor = new predictor_service_1.PredictorService(registry);
+            const svc = new metrics_service_1.MetricsService(registry, predictor);
+            const result = await svc.evaluate('recall-model', [
+                { text: 'a', label: 'positive' },
+                { text: 'b', label: 'positive' },
+                { text: 'c', label: 'negative' },
+            ], { labelKey: 'label', predictionKey: 'result', metrics: ['recall'] });
+            expect(result.metrics.recall).toBeLessThanOrEqual(1);
+            expect(result.metrics.recall).toBeGreaterThanOrEqual(0);
+        });
+    });
+    describe('recordEvaluation edge cases', () => {
+        it('should handle multiple evaluations for same model', () => {
+            service.recordEvaluation({
+                modelName: 'model',
+                version: '1.0.0',
+                metrics: { accuracy: 0.9 },
+                evaluatedAt: new Date(),
+            });
+            service.recordEvaluation({
+                modelName: 'model',
+                version: '1.0.0',
+                metrics: { accuracy: 0.95 },
+                evaluatedAt: new Date(),
+            });
+            const history = service.getHistory('model');
+            expect(history).toHaveLength(2);
+        });
+        it('should handle evaluations with custom metrics', () => {
+            service.recordEvaluation({
+                modelName: 'model',
+                version: '1.0.0',
+                metrics: { customMetric: 0.85, anotherMetric: 0.92 },
+                evaluatedAt: new Date(),
+            });
+            const metrics = service.getMetrics('model', '1.0.0');
+            expect(metrics?.metrics.customMetric).toBe(0.85);
+            expect(metrics?.metrics.anotherMetric).toBe(0.92);
+        });
     });
 });

package/dist/experiments/__tests__/experiment.decorator.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=experiment.decorator.test.d.ts.map

package/dist/experiments/__tests__/experiment.decorator.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.decorator.test.d.ts","sourceRoot":"","sources":["../../../src/experiments/__tests__/experiment.decorator.test.ts"],"names":[],"mappings":""}

package/dist/experiments/__tests__/experiment.decorator.test.js

@@ -0,0 +1,121 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const experiment_decorator_1 = require("../experiment.decorator");
+describe('Experiment decorator', () => {
+    it('should attach metadata to class', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({
+                name: 'test-experiment',
+                description: 'Test experiment',
+                tags: ['test', 'ml'],
+            })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata).toBeDefined();
+        expect(metadata?.name).toBe('test-experiment');
+        expect(metadata?.description).toBe('Test experiment');
+        expect(metadata?.tags).toEqual(['test', 'ml']);
+    });
+    it('should use class name as default experiment name', () => {
+        let MyExperiment = class MyExperiment {
+        };
+        MyExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)()
+        ], MyExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(MyExperiment);
+        expect(metadata?.name).toBe('MyExperiment');
+    });
+    it('should set default autoLogParams to true', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)()
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.autoLogParams).toBe(true);
+    });
+    it('should set default autoLogMetrics to true', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)()
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.autoLogMetrics).toBe(true);
+    });
+    it('should allow disabling autoLogParams', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ autoLogParams: false })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.autoLogParams).toBe(false);
+    });
+    it('should allow disabling autoLogMetrics', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ autoLogMetrics: false })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.autoLogMetrics).toBe(false);
+    });
+    it('should handle optional description', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ name: 'test' })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.description).toBeUndefined();
+    });
+    it('should handle optional tags', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ name: 'test' })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.tags).toBeUndefined();
+    });
+    it('should handle empty tags array', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ name: 'test', tags: [] })
+        ], TestExperiment);
+        const metadata = (0, experiment_decorator_1.getExperimentMetadata)(TestExperiment);
+        expect(metadata?.tags).toEqual([]);
+    });
+});
+describe('hasExperimentMetadata', () => {
+    it('should return true for decorated class', () => {
+        let TestExperiment = class TestExperiment {
+        };
+        TestExperiment = __decorate([
+            (0, experiment_decorator_1.Experiment)({ name: 'test' })
+        ], TestExperiment);
+        expect((0, experiment_decorator_1.hasExperimentMetadata)(TestExperiment)).toBe(true);
+    });
+    it('should return false for non-decorated class', () => {
+        class TestExperiment {
+        }
+        expect((0, experiment_decorator_1.hasExperimentMetadata)(TestExperiment)).toBe(false);
+    });
+});
+describe('getExperimentMetadata', () => {
+    it('should return undefined for non-decorated class', () => {
+        class TestExperiment {
+        }
+        expect((0, experiment_decorator_1.getExperimentMetadata)(TestExperiment)).toBeUndefined();
+    });
+});

package/dist/experiments/__tests__/experiment.service.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=experiment.service.test.d.ts.map

package/dist/experiments/__tests__/experiment.service.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.service.test.d.ts","sourceRoot":"","sources":["../../../src/experiments/__tests__/experiment.service.test.ts"],"names":[],"mappings":""}