@hazeljs/ml 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +192 -0
- package/README.md +300 -0
- package/dist/decorators/index.d.ts +4 -0
- package/dist/decorators/index.d.ts.map +1 -0
- package/dist/decorators/index.js +15 -0
- package/dist/decorators/model.decorator.d.ts +26 -0
- package/dist/decorators/model.decorator.d.ts.map +1 -0
- package/dist/decorators/model.decorator.js +48 -0
- package/dist/decorators/model.decorator.test.d.ts +2 -0
- package/dist/decorators/model.decorator.test.d.ts.map +1 -0
- package/dist/decorators/model.decorator.test.js +128 -0
- package/dist/decorators/predict.decorator.d.ts +20 -0
- package/dist/decorators/predict.decorator.d.ts.map +1 -0
- package/dist/decorators/predict.decorator.js +40 -0
- package/dist/decorators/train.decorator.d.ts +21 -0
- package/dist/decorators/train.decorator.d.ts.map +1 -0
- package/dist/decorators/train.decorator.js +41 -0
- package/dist/evaluation/metrics.service.d.ts +54 -0
- package/dist/evaluation/metrics.service.d.ts.map +1 -0
- package/dist/evaluation/metrics.service.js +163 -0
- package/dist/evaluation/metrics.service.test.d.ts +2 -0
- package/dist/evaluation/metrics.service.test.d.ts.map +1 -0
- package/dist/evaluation/metrics.service.test.js +253 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +42 -0
- package/dist/inference/batch.service.d.ts +16 -0
- package/dist/inference/batch.service.d.ts.map +1 -0
- package/dist/inference/batch.service.js +52 -0
- package/dist/inference/batch.service.test.d.ts +2 -0
- package/dist/inference/batch.service.test.d.ts.map +1 -0
- package/dist/inference/batch.service.test.js +86 -0
- package/dist/inference/predictor.service.d.ts +13 -0
- package/dist/inference/predictor.service.d.ts.map +1 -0
- package/dist/inference/predictor.service.js +65 -0
- package/dist/inference/predictor.service.test.d.ts +2 -0
- package/dist/inference/predictor.service.test.d.ts.map +1 -0
- package/dist/inference/predictor.service.test.js +115 -0
- package/dist/ml-model.base.d.ts +20 -0
- package/dist/ml-model.base.d.ts.map +1 -0
- package/dist/ml-model.base.js +33 -0
- package/dist/ml-model.base.test.d.ts +2 -0
- package/dist/ml-model.base.test.d.ts.map +1 -0
- package/dist/ml-model.base.test.js +57 -0
- package/dist/ml.module.d.ts +27 -0
- package/dist/ml.module.d.ts.map +1 -0
- package/dist/ml.module.js +126 -0
- package/dist/ml.module.test.d.ts +2 -0
- package/dist/ml.module.test.d.ts.map +1 -0
- package/dist/ml.module.test.js +60 -0
- package/dist/ml.types.d.ts +30 -0
- package/dist/ml.types.d.ts.map +1 -0
- package/dist/ml.types.js +5 -0
- package/dist/registry/model.registry.d.ts +21 -0
- package/dist/registry/model.registry.d.ts.map +1 -0
- package/dist/registry/model.registry.js +64 -0
- package/dist/registry/model.registry.test.d.ts +2 -0
- package/dist/registry/model.registry.test.d.ts.map +1 -0
- package/dist/registry/model.registry.test.js +93 -0
- package/dist/training/pipeline.service.d.ts +25 -0
- package/dist/training/pipeline.service.d.ts.map +1 -0
- package/dist/training/pipeline.service.js +65 -0
- package/dist/training/pipeline.service.test.d.ts +2 -0
- package/dist/training/pipeline.service.test.d.ts.map +1 -0
- package/dist/training/pipeline.service.test.js +52 -0
- package/dist/training/trainer.service.d.ts +13 -0
- package/dist/training/trainer.service.d.ts.map +1 -0
- package/dist/training/trainer.service.js +69 -0
- package/dist/training/trainer.service.test.d.ts +2 -0
- package/dist/training/trainer.service.test.d.ts.map +1 -0
- package/dist/training/trainer.service.test.js +99 -0
- package/package.json +52 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
require("reflect-metadata");
|
|
13
|
+
const model_decorator_1 = require("./model.decorator");
|
|
14
|
+
const train_decorator_1 = require("./train.decorator");
|
|
15
|
+
const predict_decorator_1 = require("./predict.decorator");
|
|
16
|
+
describe('Model decorator', () => {
|
|
17
|
+
it('applies metadata to class', () => {
|
|
18
|
+
let TestModel = class TestModel {
|
|
19
|
+
};
|
|
20
|
+
TestModel = __decorate([
|
|
21
|
+
(0, model_decorator_1.Model)({ name: 'test-model', version: '1.0.0', framework: 'tensorflow' })
|
|
22
|
+
], TestModel);
|
|
23
|
+
const meta = (0, model_decorator_1.getModelMetadata)(TestModel);
|
|
24
|
+
expect(meta).toEqual({
|
|
25
|
+
name: 'test-model',
|
|
26
|
+
version: '1.0.0',
|
|
27
|
+
framework: 'tensorflow',
|
|
28
|
+
description: '',
|
|
29
|
+
tags: [],
|
|
30
|
+
});
|
|
31
|
+
expect((0, model_decorator_1.hasModelMetadata)(TestModel)).toBe(true);
|
|
32
|
+
});
|
|
33
|
+
it('merges optional fields', () => {
|
|
34
|
+
let CustomModel = class CustomModel {
|
|
35
|
+
};
|
|
36
|
+
CustomModel = __decorate([
|
|
37
|
+
(0, model_decorator_1.Model)({
|
|
38
|
+
name: 'custom',
|
|
39
|
+
version: '2.0.0',
|
|
40
|
+
framework: 'onnx',
|
|
41
|
+
description: 'A model',
|
|
42
|
+
tags: ['production'],
|
|
43
|
+
})
|
|
44
|
+
], CustomModel);
|
|
45
|
+
const meta = (0, model_decorator_1.getModelMetadata)(CustomModel);
|
|
46
|
+
expect(meta?.description).toBe('A model');
|
|
47
|
+
expect(meta?.tags).toEqual(['production']);
|
|
48
|
+
});
|
|
49
|
+
it('hasModelMetadata returns false for undecorated class', () => {
|
|
50
|
+
class PlainClass {
|
|
51
|
+
}
|
|
52
|
+
expect((0, model_decorator_1.hasModelMetadata)(PlainClass)).toBe(false);
|
|
53
|
+
expect((0, model_decorator_1.getModelMetadata)(PlainClass)).toBeUndefined();
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
describe('Train decorator', () => {
|
|
57
|
+
it('applies metadata to method', () => {
|
|
58
|
+
class TestClass {
|
|
59
|
+
train() { }
|
|
60
|
+
}
|
|
61
|
+
__decorate([
|
|
62
|
+
(0, train_decorator_1.Train)({ pipeline: 'default', epochs: 5 }),
|
|
63
|
+
__metadata("design:type", Function),
|
|
64
|
+
__metadata("design:paramtypes", []),
|
|
65
|
+
__metadata("design:returntype", void 0)
|
|
66
|
+
], TestClass.prototype, "train", null);
|
|
67
|
+
const meta = (0, train_decorator_1.getTrainMetadata)(TestClass.prototype, 'train');
|
|
68
|
+
expect(meta).toEqual({
|
|
69
|
+
pipeline: 'default',
|
|
70
|
+
batchSize: 32,
|
|
71
|
+
epochs: 5,
|
|
72
|
+
});
|
|
73
|
+
expect((0, train_decorator_1.hasTrainMetadata)(TestClass.prototype, 'train')).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
it('uses defaults when no options', () => {
|
|
76
|
+
class TestClass {
|
|
77
|
+
train() { }
|
|
78
|
+
}
|
|
79
|
+
__decorate([
|
|
80
|
+
(0, train_decorator_1.Train)(),
|
|
81
|
+
__metadata("design:type", Function),
|
|
82
|
+
__metadata("design:paramtypes", []),
|
|
83
|
+
__metadata("design:returntype", void 0)
|
|
84
|
+
], TestClass.prototype, "train", null);
|
|
85
|
+
const meta = (0, train_decorator_1.getTrainMetadata)(TestClass.prototype, 'train');
|
|
86
|
+
expect(meta?.batchSize).toBe(32);
|
|
87
|
+
expect(meta?.epochs).toBe(10);
|
|
88
|
+
});
|
|
89
|
+
it('hasTrainMetadata returns false for undecorated method', () => {
|
|
90
|
+
class TestClass {
|
|
91
|
+
train() { }
|
|
92
|
+
}
|
|
93
|
+
expect((0, train_decorator_1.hasTrainMetadata)(TestClass.prototype, 'train')).toBe(false);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
describe('Predict decorator', () => {
|
|
97
|
+
it('applies metadata to method', () => {
|
|
98
|
+
class TestClass {
|
|
99
|
+
predict() { }
|
|
100
|
+
}
|
|
101
|
+
__decorate([
|
|
102
|
+
(0, predict_decorator_1.Predict)({ endpoint: '/predict', batch: true }),
|
|
103
|
+
__metadata("design:type", Function),
|
|
104
|
+
__metadata("design:paramtypes", []),
|
|
105
|
+
__metadata("design:returntype", void 0)
|
|
106
|
+
], TestClass.prototype, "predict", null);
|
|
107
|
+
const meta = (0, predict_decorator_1.getPredictMetadata)(TestClass.prototype, 'predict');
|
|
108
|
+
expect(meta).toEqual({
|
|
109
|
+
batch: true,
|
|
110
|
+
endpoint: '/predict',
|
|
111
|
+
});
|
|
112
|
+
expect((0, predict_decorator_1.hasPredictMetadata)(TestClass.prototype, 'predict')).toBe(true);
|
|
113
|
+
});
|
|
114
|
+
it('uses defaults when no options', () => {
|
|
115
|
+
class TestClass {
|
|
116
|
+
predict() { }
|
|
117
|
+
}
|
|
118
|
+
__decorate([
|
|
119
|
+
(0, predict_decorator_1.Predict)(),
|
|
120
|
+
__metadata("design:type", Function),
|
|
121
|
+
__metadata("design:paramtypes", []),
|
|
122
|
+
__metadata("design:returntype", void 0)
|
|
123
|
+
], TestClass.prototype, "predict", null);
|
|
124
|
+
const meta = (0, predict_decorator_1.getPredictMetadata)(TestClass.prototype, 'predict');
|
|
125
|
+
expect(meta?.batch).toBe(false);
|
|
126
|
+
expect(meta?.endpoint).toBe('/predict');
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import 'reflect-metadata';
|
|
2
|
+
export interface PredictOptions {
|
|
3
|
+
batch?: boolean;
|
|
4
|
+
endpoint?: string;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* @Predict decorator - Mark a method as the inference endpoint
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```typescript
|
|
11
|
+
* @Predict({ endpoint: '/predict' })
|
|
12
|
+
* async predict(text: string) {
|
|
13
|
+
* return { sentiment: 'positive', confidence: 0.92 };
|
|
14
|
+
* }
|
|
15
|
+
* ```
|
|
16
|
+
*/
|
|
17
|
+
export declare function Predict(options?: PredictOptions): MethodDecorator;
|
|
18
|
+
export declare function getPredictMetadata(target: object, propertyKey: string | symbol): PredictOptions | undefined;
|
|
19
|
+
export declare function hasPredictMetadata(target: object, propertyKey: string | symbol): boolean;
|
|
20
|
+
//# sourceMappingURL=predict.decorator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"predict.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/predict.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAK1B,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,eAAe,CAarE;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,cAAc,GAAG,SAAS,CAE5B;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAExF"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Predict = Predict;
|
|
7
|
+
exports.getPredictMetadata = getPredictMetadata;
|
|
8
|
+
exports.hasPredictMetadata = hasPredictMetadata;
|
|
9
|
+
require("reflect-metadata");
|
|
10
|
+
const core_1 = __importDefault(require("@hazeljs/core"));
|
|
11
|
+
const PREDICT_METADATA_KEY = 'hazel:ml:predict';
|
|
12
|
+
/**
|
|
13
|
+
* @Predict decorator - Mark a method as the inference endpoint
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```typescript
|
|
17
|
+
* @Predict({ endpoint: '/predict' })
|
|
18
|
+
* async predict(text: string) {
|
|
19
|
+
* return { sentiment: 'positive', confidence: 0.92 };
|
|
20
|
+
* }
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
function Predict(options = {}) {
|
|
24
|
+
return (target, propertyKey, descriptor) => {
|
|
25
|
+
const metadata = {
|
|
26
|
+
batch: false,
|
|
27
|
+
endpoint: '/predict',
|
|
28
|
+
...options,
|
|
29
|
+
};
|
|
30
|
+
Reflect.defineMetadata(PREDICT_METADATA_KEY, metadata, target, propertyKey);
|
|
31
|
+
core_1.default.debug(`Predict decorator applied to ${target.constructor.name}.${String(propertyKey)}`);
|
|
32
|
+
return descriptor;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function getPredictMetadata(target, propertyKey) {
|
|
36
|
+
return Reflect.getMetadata(PREDICT_METADATA_KEY, target, propertyKey);
|
|
37
|
+
}
|
|
38
|
+
function hasPredictMetadata(target, propertyKey) {
|
|
39
|
+
return Reflect.hasMetadata(PREDICT_METADATA_KEY, target, propertyKey);
|
|
40
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import 'reflect-metadata';
|
|
2
|
+
export interface TrainOptions {
|
|
3
|
+
pipeline?: string;
|
|
4
|
+
batchSize?: number;
|
|
5
|
+
epochs?: number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* @Train decorator - Mark a method as the training pipeline
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* @Train({ pipeline: 'default', epochs: 10 })
|
|
13
|
+
* async train(data: TrainingData) {
|
|
14
|
+
* return { accuracy: 0.95, loss: 0.05 };
|
|
15
|
+
* }
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
export declare function Train(options?: TrainOptions): MethodDecorator;
|
|
19
|
+
export declare function getTrainMetadata(target: object, propertyKey: string | symbol): TrainOptions | undefined;
|
|
20
|
+
export declare function hasTrainMetadata(target: object, propertyKey: string | symbol): boolean;
|
|
21
|
+
//# sourceMappingURL=train.decorator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"train.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/train.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAK1B,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,KAAK,CAAC,OAAO,GAAE,YAAiB,GAAG,eAAe,CAcjE;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,YAAY,GAAG,SAAS,CAE1B;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAEtF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Train = Train;
|
|
7
|
+
exports.getTrainMetadata = getTrainMetadata;
|
|
8
|
+
exports.hasTrainMetadata = hasTrainMetadata;
|
|
9
|
+
require("reflect-metadata");
|
|
10
|
+
const core_1 = __importDefault(require("@hazeljs/core"));
|
|
11
|
+
const TRAIN_METADATA_KEY = 'hazel:ml:train';
|
|
12
|
+
/**
|
|
13
|
+
* @Train decorator - Mark a method as the training pipeline
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```typescript
|
|
17
|
+
* @Train({ pipeline: 'default', epochs: 10 })
|
|
18
|
+
* async train(data: TrainingData) {
|
|
19
|
+
* return { accuracy: 0.95, loss: 0.05 };
|
|
20
|
+
* }
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
function Train(options = {}) {
|
|
24
|
+
return (target, propertyKey, descriptor) => {
|
|
25
|
+
const metadata = {
|
|
26
|
+
pipeline: 'default',
|
|
27
|
+
batchSize: 32,
|
|
28
|
+
epochs: 10,
|
|
29
|
+
...options,
|
|
30
|
+
};
|
|
31
|
+
Reflect.defineMetadata(TRAIN_METADATA_KEY, metadata, target, propertyKey);
|
|
32
|
+
core_1.default.debug(`Train decorator applied to ${target.constructor.name}.${String(propertyKey)}`);
|
|
33
|
+
return descriptor;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function getTrainMetadata(target, propertyKey) {
|
|
37
|
+
return Reflect.getMetadata(TRAIN_METADATA_KEY, target, propertyKey);
|
|
38
|
+
}
|
|
39
|
+
function hasTrainMetadata(target, propertyKey) {
|
|
40
|
+
return Reflect.hasMetadata(TRAIN_METADATA_KEY, target, propertyKey);
|
|
41
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { ModelRegistry } from '../registry/model.registry';
|
|
2
|
+
import { PredictorService } from '../inference/predictor.service';
|
|
3
|
+
export interface ModelMetrics {
|
|
4
|
+
accuracy?: number;
|
|
5
|
+
precision?: number;
|
|
6
|
+
recall?: number;
|
|
7
|
+
f1Score?: number;
|
|
8
|
+
loss?: number;
|
|
9
|
+
[key: string]: number | undefined;
|
|
10
|
+
}
|
|
11
|
+
export interface EvaluationResult {
|
|
12
|
+
modelName: string;
|
|
13
|
+
version: string;
|
|
14
|
+
metrics: ModelMetrics;
|
|
15
|
+
evaluatedAt: Date;
|
|
16
|
+
}
|
|
17
|
+
export type EvaluateMetric = 'accuracy' | 'f1' | 'precision' | 'recall';
|
|
18
|
+
export interface EvaluateOptions {
|
|
19
|
+
metrics?: EvaluateMetric[];
|
|
20
|
+
labelKey?: string;
|
|
21
|
+
predictionKey?: string;
|
|
22
|
+
version?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Metrics Service - Model evaluation and metrics
|
|
26
|
+
* Tracks model performance for A/B testing and monitoring
|
|
27
|
+
*/
|
|
28
|
+
export declare class MetricsService {
|
|
29
|
+
private readonly modelRegistry?;
|
|
30
|
+
private readonly predictorService?;
|
|
31
|
+
constructor(modelRegistry?: ModelRegistry | undefined, predictorService?: PredictorService | undefined);
|
|
32
|
+
private metrics;
|
|
33
|
+
recordEvaluation(result: EvaluationResult): void;
|
|
34
|
+
getMetrics(modelName: string, version?: string): EvaluationResult | undefined;
|
|
35
|
+
getHistory(modelName: string): EvaluationResult[];
|
|
36
|
+
compareVersions(modelName: string, versionA: string, versionB: string): {
|
|
37
|
+
a: EvaluationResult | undefined;
|
|
38
|
+
b: EvaluationResult | undefined;
|
|
39
|
+
winner?: string;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Evaluate model on test data by running predictions and computing metrics.
|
|
43
|
+
* Requires PredictorService to be injected.
|
|
44
|
+
*
|
|
45
|
+
* @param modelName - Registered model name
|
|
46
|
+
* @param testData - Array of samples. Each sample must contain the model input and a label key.
|
|
47
|
+
* @param options - labelKey (default: 'label'), predictionKey (tries 'label'|'sentiment'|'class'), metrics, version
|
|
48
|
+
*/
|
|
49
|
+
evaluate(modelName: string, testData: Record<string, unknown>[], options?: EvaluateOptions): Promise<EvaluationResult>;
|
|
50
|
+
private extractPredictedLabel;
|
|
51
|
+
private computeAccuracy;
|
|
52
|
+
private computePrecisionRecallF1;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=metrics.service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.service.d.ts","sourceRoot":"","sources":["../../src/evaluation/metrics.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAIlE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,YAAY,CAAC;IACtB,WAAW,EAAE,IAAI,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,IAAI,GAAG,WAAW,GAAG,QAAQ,CAAC;AAExE,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,qBACa,cAAc;IAEvB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBADjB,aAAa,CAAC,EAAE,aAAa,YAAA,EAC7B,gBAAgB,CAAC,EAAE,gBAAgB,YAAA;IAGtD,OAAO,CAAC,OAAO,CAA8C;IAE7D,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAQhD,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAQ7E,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,EAAE;IAIjD,eAAe,CACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf;QACD,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;QAChC,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;IAaD;;;;;;;OAOG;IACG,QAAQ,CACZ,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACnC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,gBAAgB,CAAC;IA2D5B,OAAO,CAAC,qBAAqB;IAW7B,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,wBAAwB;CAmCjC"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.MetricsService = void 0;
|
|
16
|
+
const core_1 = require("@hazeljs/core");
|
|
17
|
+
const model_registry_1 = require("../registry/model.registry");
|
|
18
|
+
const predictor_service_1 = require("../inference/predictor.service");
|
|
19
|
+
const core_2 = __importDefault(require("@hazeljs/core"));
|
|
20
|
+
/**
|
|
21
|
+
* Metrics Service - Model evaluation and metrics
|
|
22
|
+
* Tracks model performance for A/B testing and monitoring
|
|
23
|
+
*/
|
|
24
|
+
let MetricsService = class MetricsService {
|
|
25
|
+
constructor(modelRegistry, predictorService) {
|
|
26
|
+
this.modelRegistry = modelRegistry;
|
|
27
|
+
this.predictorService = predictorService;
|
|
28
|
+
this.metrics = new Map();
|
|
29
|
+
}
|
|
30
|
+
recordEvaluation(result) {
|
|
31
|
+
const key = result.modelName;
|
|
32
|
+
const existing = this.metrics.get(key) || [];
|
|
33
|
+
existing.push(result);
|
|
34
|
+
this.metrics.set(key, existing);
|
|
35
|
+
core_2.default.debug(`Recorded evaluation for ${key}@${result.version}`);
|
|
36
|
+
}
|
|
37
|
+
getMetrics(modelName, version) {
|
|
38
|
+
const results = this.metrics.get(modelName) || [];
|
|
39
|
+
if (version) {
|
|
40
|
+
return results.find((r) => r.version === version);
|
|
41
|
+
}
|
|
42
|
+
return results[results.length - 1];
|
|
43
|
+
}
|
|
44
|
+
getHistory(modelName) {
|
|
45
|
+
return this.metrics.get(modelName) || [];
|
|
46
|
+
}
|
|
47
|
+
compareVersions(modelName, versionA, versionB) {
|
|
48
|
+
const results = this.metrics.get(modelName) || [];
|
|
49
|
+
const a = results.find((r) => r.version === versionA);
|
|
50
|
+
const b = results.find((r) => r.version === versionB);
|
|
51
|
+
let winner;
|
|
52
|
+
if (a?.metrics.accuracy !== undefined && b?.metrics.accuracy !== undefined) {
|
|
53
|
+
winner = a.metrics.accuracy >= b.metrics.accuracy ? versionA : versionB;
|
|
54
|
+
}
|
|
55
|
+
return { a, b, winner };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Evaluate model on test data by running predictions and computing metrics.
|
|
59
|
+
* Requires PredictorService to be injected.
|
|
60
|
+
*
|
|
61
|
+
* @param modelName - Registered model name
|
|
62
|
+
* @param testData - Array of samples. Each sample must contain the model input and a label key.
|
|
63
|
+
* @param options - labelKey (default: 'label'), predictionKey (tries 'label'|'sentiment'|'class'), metrics, version
|
|
64
|
+
*/
|
|
65
|
+
async evaluate(modelName, testData, options = {}) {
|
|
66
|
+
if (!this.predictorService) {
|
|
67
|
+
throw new Error('MetricsService.evaluate() requires PredictorService. Ensure MLModule is configured with PredictorService.');
|
|
68
|
+
}
|
|
69
|
+
const { metrics: requestedMetrics = ['accuracy', 'f1', 'precision', 'recall'], labelKey = 'label', predictionKey, version, } = options;
|
|
70
|
+
if (testData.length === 0) {
|
|
71
|
+
throw new Error('testData cannot be empty');
|
|
72
|
+
}
|
|
73
|
+
const predictions = [];
|
|
74
|
+
for (const sample of testData) {
|
|
75
|
+
const { [labelKey]: _label, ...input } = sample;
|
|
76
|
+
const pred = await this.predictorService.predict(modelName, input, version);
|
|
77
|
+
predictions.push(pred);
|
|
78
|
+
}
|
|
79
|
+
const labels = testData.map((s) => String(s[labelKey] ?? ''));
|
|
80
|
+
const predictedLabels = predictions.map((p) => this.extractPredictedLabel(p, predictionKey));
|
|
81
|
+
const computed = {};
|
|
82
|
+
if (requestedMetrics.includes('accuracy')) {
|
|
83
|
+
computed.accuracy = this.computeAccuracy(labels, predictedLabels);
|
|
84
|
+
}
|
|
85
|
+
if (requestedMetrics.includes('precision') ||
|
|
86
|
+
requestedMetrics.includes('recall') ||
|
|
87
|
+
requestedMetrics.includes('f1')) {
|
|
88
|
+
const { precision, recall, f1Score } = this.computePrecisionRecallF1(labels, predictedLabels);
|
|
89
|
+
if (requestedMetrics.includes('precision'))
|
|
90
|
+
computed.precision = precision;
|
|
91
|
+
if (requestedMetrics.includes('recall'))
|
|
92
|
+
computed.recall = recall;
|
|
93
|
+
if (requestedMetrics.includes('f1'))
|
|
94
|
+
computed.f1Score = f1Score;
|
|
95
|
+
}
|
|
96
|
+
const model = this.modelRegistry?.get(modelName, version);
|
|
97
|
+
const modelVersion = model?.metadata.version ?? version ?? 'unknown';
|
|
98
|
+
const result = {
|
|
99
|
+
modelName,
|
|
100
|
+
version: modelVersion,
|
|
101
|
+
metrics: computed,
|
|
102
|
+
evaluatedAt: new Date(),
|
|
103
|
+
};
|
|
104
|
+
this.recordEvaluation(result);
|
|
105
|
+
core_2.default.debug(`Evaluated ${modelName}@${modelVersion}`, computed);
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
extractPredictedLabel(prediction, key) {
|
|
109
|
+
if (key && prediction[key] !== undefined) {
|
|
110
|
+
return String(prediction[key]);
|
|
111
|
+
}
|
|
112
|
+
for (const k of ['label', 'sentiment', 'class', 'prediction']) {
|
|
113
|
+
if (prediction[k] !== undefined)
|
|
114
|
+
return String(prediction[k]);
|
|
115
|
+
}
|
|
116
|
+
const first = Object.values(prediction)[0];
|
|
117
|
+
return first !== undefined ? String(first) : '';
|
|
118
|
+
}
|
|
119
|
+
computeAccuracy(labels, predicted) {
|
|
120
|
+
let correct = 0;
|
|
121
|
+
for (let i = 0; i < labels.length; i++) {
|
|
122
|
+
if (labels[i] === predicted[i])
|
|
123
|
+
correct++;
|
|
124
|
+
}
|
|
125
|
+
return labels.length > 0 ? correct / labels.length : 0;
|
|
126
|
+
}
|
|
127
|
+
computePrecisionRecallF1(labels, predicted) {
|
|
128
|
+
const classes = [...new Set([...labels, ...predicted])].filter(Boolean);
|
|
129
|
+
if (classes.length === 0)
|
|
130
|
+
return { precision: 0, recall: 0, f1Score: 0 };
|
|
131
|
+
let totalPrecision = 0;
|
|
132
|
+
let totalRecall = 0;
|
|
133
|
+
let count = 0;
|
|
134
|
+
for (const cls of classes) {
|
|
135
|
+
let tp = 0, fp = 0, fn = 0;
|
|
136
|
+
for (let i = 0; i < labels.length; i++) {
|
|
137
|
+
const isPred = predicted[i] === cls;
|
|
138
|
+
const isActual = labels[i] === cls;
|
|
139
|
+
if (isPred && isActual)
|
|
140
|
+
tp++;
|
|
141
|
+
if (isPred && !isActual)
|
|
142
|
+
fp++;
|
|
143
|
+
if (!isPred && isActual)
|
|
144
|
+
fn++;
|
|
145
|
+
}
|
|
146
|
+
const precision = tp + fp > 0 ? tp / (tp + fp) : 0;
|
|
147
|
+
const recall = tp + fn > 0 ? tp / (tp + fn) : 0;
|
|
148
|
+
totalPrecision += precision;
|
|
149
|
+
totalRecall += recall;
|
|
150
|
+
count++;
|
|
151
|
+
}
|
|
152
|
+
const precision = count > 0 ? totalPrecision / count : 0;
|
|
153
|
+
const recall = count > 0 ? totalRecall / count : 0;
|
|
154
|
+
const f1Score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
|
|
155
|
+
return { precision, recall, f1Score };
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
exports.MetricsService = MetricsService;
|
|
159
|
+
exports.MetricsService = MetricsService = __decorate([
|
|
160
|
+
(0, core_1.Service)(),
|
|
161
|
+
__metadata("design:paramtypes", [model_registry_1.ModelRegistry,
|
|
162
|
+
predictor_service_1.PredictorService])
|
|
163
|
+
], MetricsService);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.service.test.d.ts","sourceRoot":"","sources":["../../src/evaluation/metrics.service.test.ts"],"names":[],"mappings":""}
|