@hazeljs/ml 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evaluation/metrics.service.test.js +288 -0
- package/dist/experiments/__tests__/experiment.decorator.test.d.ts +2 -0
- package/dist/experiments/__tests__/experiment.decorator.test.d.ts.map +1 -0
- package/dist/experiments/__tests__/experiment.decorator.test.js +121 -0
- package/dist/experiments/__tests__/experiment.service.test.d.ts +2 -0
- package/dist/experiments/__tests__/experiment.service.test.d.ts.map +1 -0
- package/dist/experiments/__tests__/experiment.service.test.js +460 -0
- package/dist/experiments/experiment.decorator.d.ts +44 -0
- package/dist/experiments/experiment.decorator.d.ts.map +1 -0
- package/dist/experiments/experiment.decorator.js +51 -0
- package/dist/experiments/experiment.service.d.ts +42 -0
- package/dist/experiments/experiment.service.d.ts.map +1 -0
- package/dist/experiments/experiment.service.js +355 -0
- package/dist/experiments/experiment.types.d.ts +60 -0
- package/dist/experiments/experiment.types.d.ts.map +1 -0
- package/dist/experiments/experiment.types.js +5 -0
- package/dist/experiments/index.d.ts +9 -0
- package/dist/experiments/index.d.ts.map +1 -0
- package/dist/experiments/index.js +16 -0
- package/dist/features/__tests__/feature-view.decorator.test.d.ts +2 -0
- package/dist/features/__tests__/feature-view.decorator.test.d.ts.map +1 -0
- package/dist/features/__tests__/feature-view.decorator.test.js +168 -0
- package/dist/features/__tests__/feature.decorator.test.d.ts +2 -0
- package/dist/features/__tests__/feature.decorator.test.d.ts.map +1 -0
- package/dist/features/__tests__/feature.decorator.test.js +167 -0
- package/dist/features/feature-store.service.d.ts +59 -0
- package/dist/features/feature-store.service.d.ts.map +1 -0
- package/dist/features/feature-store.service.js +197 -0
- package/dist/features/feature-view.decorator.d.ts +52 -0
- package/dist/features/feature-view.decorator.d.ts.map +1 -0
- package/dist/features/feature-view.decorator.js +54 -0
- package/dist/features/feature.decorator.d.ts +42 -0
- package/dist/features/feature.decorator.d.ts.map +1 -0
- package/dist/features/feature.decorator.js +49 -0
- package/dist/features/feature.types.d.ts +93 -0
- package/dist/features/feature.types.d.ts.map +1 -0
- package/dist/features/feature.types.js +5 -0
- package/dist/features/index.d.ts +12 -0
- package/dist/features/index.d.ts.map +1 -0
- package/dist/features/index.js +29 -0
- package/dist/features/offline-store.d.ts +40 -0
- package/dist/features/offline-store.d.ts.map +1 -0
- package/dist/features/offline-store.js +215 -0
- package/dist/features/online-store.d.ts +45 -0
- package/dist/features/online-store.d.ts.map +1 -0
- package/dist/features/online-store.js +139 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +26 -1
- package/dist/monitoring/__tests__/drift.service.test.d.ts +2 -0
- package/dist/monitoring/__tests__/drift.service.test.d.ts.map +1 -0
- package/dist/monitoring/__tests__/drift.service.test.js +362 -0
- package/dist/monitoring/__tests__/monitor.service.test.d.ts +2 -0
- package/dist/monitoring/__tests__/monitor.service.test.d.ts.map +1 -0
- package/dist/monitoring/__tests__/monitor.service.test.js +360 -0
- package/dist/monitoring/drift.service.d.ts +68 -0
- package/dist/monitoring/drift.service.d.ts.map +1 -0
- package/dist/monitoring/drift.service.js +360 -0
- package/dist/monitoring/drift.types.d.ts +44 -0
- package/dist/monitoring/drift.types.d.ts.map +1 -0
- package/dist/monitoring/drift.types.js +5 -0
- package/dist/monitoring/index.d.ts +10 -0
- package/dist/monitoring/index.d.ts.map +1 -0
- package/dist/monitoring/index.js +13 -0
- package/dist/monitoring/monitor.service.d.ts +79 -0
- package/dist/monitoring/monitor.service.d.ts.map +1 -0
- package/dist/monitoring/monitor.service.js +192 -0
- package/dist/training/trainer.service.test.js +105 -0
- package/package.json +2 -2
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const experiment_service_1 = require("../experiment.service");
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const path_1 = require("path");
|
|
6
|
+
describe('ExperimentService', () => {
|
|
7
|
+
let service;
|
|
8
|
+
const testStorageDir = (0, path_1.join)(__dirname, '__test_experiments__');
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
service = new experiment_service_1.ExperimentService();
|
|
11
|
+
});
|
|
12
|
+
afterEach(() => {
|
|
13
|
+
if ((0, fs_1.existsSync)(testStorageDir)) {
|
|
14
|
+
(0, fs_1.rmSync)(testStorageDir, { recursive: true, force: true });
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
describe('configure', () => {
|
|
18
|
+
it('should configure with memory storage', () => {
|
|
19
|
+
service.configure({ storage: 'memory' });
|
|
20
|
+
expect(() => service.listExperiments()).not.toThrow();
|
|
21
|
+
});
|
|
22
|
+
it('should configure with file storage', () => {
|
|
23
|
+
service.configure({
|
|
24
|
+
storage: 'file',
|
|
25
|
+
file: { directory: testStorageDir },
|
|
26
|
+
});
|
|
27
|
+
expect(() => service.listExperiments()).not.toThrow();
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
describe('createExperiment', () => {
|
|
31
|
+
it('should create experiment with name', () => {
|
|
32
|
+
const exp = service.createExperiment('test-exp');
|
|
33
|
+
expect(exp.id).toBeDefined();
|
|
34
|
+
expect(exp.name).toBe('test-exp');
|
|
35
|
+
expect(exp.createdAt).toBeInstanceOf(Date);
|
|
36
|
+
expect(exp.updatedAt).toBeInstanceOf(Date);
|
|
37
|
+
});
|
|
38
|
+
it('should create experiment with description', () => {
|
|
39
|
+
const exp = service.createExperiment('test-exp', {
|
|
40
|
+
description: 'Test experiment',
|
|
41
|
+
});
|
|
42
|
+
expect(exp.description).toBe('Test experiment');
|
|
43
|
+
});
|
|
44
|
+
it('should create experiment with tags', () => {
|
|
45
|
+
const exp = service.createExperiment('test-exp', {
|
|
46
|
+
tags: ['ml', 'test'],
|
|
47
|
+
});
|
|
48
|
+
expect(exp.tags).toEqual(['ml', 'test']);
|
|
49
|
+
});
|
|
50
|
+
it('should create experiment without optional fields', () => {
|
|
51
|
+
const exp = service.createExperiment('test-exp');
|
|
52
|
+
expect(exp.description).toBeUndefined();
|
|
53
|
+
expect(exp.tags).toBeUndefined();
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
describe('getExperiment', () => {
|
|
57
|
+
it('should return experiment by id', () => {
|
|
58
|
+
const exp = service.createExperiment('test-exp');
|
|
59
|
+
const retrieved = service.getExperiment(exp.id);
|
|
60
|
+
expect(retrieved).toEqual(exp);
|
|
61
|
+
});
|
|
62
|
+
it('should return undefined for non-existent experiment', () => {
|
|
63
|
+
expect(service.getExperiment('non-existent')).toBeUndefined();
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
describe('listExperiments', () => {
|
|
67
|
+
it('should return empty array when no experiments', () => {
|
|
68
|
+
expect(service.listExperiments()).toEqual([]);
|
|
69
|
+
});
|
|
70
|
+
it('should list all experiments', () => {
|
|
71
|
+
const exp1 = service.createExperiment('exp1');
|
|
72
|
+
const exp2 = service.createExperiment('exp2');
|
|
73
|
+
const experiments = service.listExperiments();
|
|
74
|
+
expect(experiments).toHaveLength(2);
|
|
75
|
+
expect(experiments).toContainEqual(exp1);
|
|
76
|
+
expect(experiments).toContainEqual(exp2);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
describe('deleteExperiment', () => {
|
|
80
|
+
it('should delete experiment', () => {
|
|
81
|
+
const exp = service.createExperiment('test-exp');
|
|
82
|
+
const deleted = service.deleteExperiment(exp.id);
|
|
83
|
+
expect(deleted).toBe(true);
|
|
84
|
+
expect(service.getExperiment(exp.id)).toBeUndefined();
|
|
85
|
+
});
|
|
86
|
+
it('should return false for non-existent experiment', () => {
|
|
87
|
+
expect(service.deleteExperiment('non-existent')).toBe(false);
|
|
88
|
+
});
|
|
89
|
+
it('should delete associated runs when deleting experiment', () => {
|
|
90
|
+
const exp = service.createExperiment('test-exp');
|
|
91
|
+
const run = service.startRun(exp.id);
|
|
92
|
+
service.deleteExperiment(exp.id);
|
|
93
|
+
expect(service.getRun(run.id)).toBeUndefined();
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
describe('startRun', () => {
|
|
97
|
+
it('should start run for experiment', () => {
|
|
98
|
+
const exp = service.createExperiment('test-exp');
|
|
99
|
+
const run = service.startRun(exp.id);
|
|
100
|
+
expect(run.id).toBeDefined();
|
|
101
|
+
expect(run.experimentId).toBe(exp.id);
|
|
102
|
+
expect(run.status).toBe('running');
|
|
103
|
+
expect(run.startTime).toBeInstanceOf(Date);
|
|
104
|
+
expect(run.params).toEqual({});
|
|
105
|
+
expect(run.metrics).toEqual({});
|
|
106
|
+
expect(run.artifacts).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
it('should start run with name', () => {
|
|
109
|
+
const exp = service.createExperiment('test-exp');
|
|
110
|
+
const run = service.startRun(exp.id, { name: 'test-run' });
|
|
111
|
+
expect(run.name).toBe('test-run');
|
|
112
|
+
});
|
|
113
|
+
it('should start run with params', () => {
|
|
114
|
+
const exp = service.createExperiment('test-exp');
|
|
115
|
+
const params = { lr: 0.001, epochs: 10 };
|
|
116
|
+
const run = service.startRun(exp.id, { params });
|
|
117
|
+
expect(run.params).toEqual(params);
|
|
118
|
+
});
|
|
119
|
+
it('should start run with tags', () => {
|
|
120
|
+
const exp = service.createExperiment('test-exp');
|
|
121
|
+
const run = service.startRun(exp.id, { tags: ['baseline', 'v1'] });
|
|
122
|
+
expect(run.tags).toEqual(['baseline', 'v1']);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
describe('endRun', () => {
|
|
126
|
+
it('should end run with completed status', () => {
|
|
127
|
+
const exp = service.createExperiment('test-exp');
|
|
128
|
+
const run = service.startRun(exp.id);
|
|
129
|
+
const endedRun = service.endRun(run.id, 'completed');
|
|
130
|
+
expect(endedRun.status).toBe('completed');
|
|
131
|
+
expect(endedRun.endTime).toBeInstanceOf(Date);
|
|
132
|
+
expect(endedRun.durationMs).toBeGreaterThanOrEqual(0);
|
|
133
|
+
});
|
|
134
|
+
it('should end run with failed status', () => {
|
|
135
|
+
const exp = service.createExperiment('test-exp');
|
|
136
|
+
const run = service.startRun(exp.id);
|
|
137
|
+
const endedRun = service.endRun(run.id, 'failed');
|
|
138
|
+
expect(endedRun.status).toBe('failed');
|
|
139
|
+
});
|
|
140
|
+
it('should end run with aborted status', () => {
|
|
141
|
+
const exp = service.createExperiment('test-exp');
|
|
142
|
+
const run = service.startRun(exp.id);
|
|
143
|
+
const endedRun = service.endRun(run.id, 'aborted');
|
|
144
|
+
expect(endedRun.status).toBe('aborted');
|
|
145
|
+
});
|
|
146
|
+
it('should default to completed status', () => {
|
|
147
|
+
const exp = service.createExperiment('test-exp');
|
|
148
|
+
const run = service.startRun(exp.id);
|
|
149
|
+
const endedRun = service.endRun(run.id);
|
|
150
|
+
expect(endedRun.status).toBe('completed');
|
|
151
|
+
});
|
|
152
|
+
it('should throw error for non-existent run', () => {
|
|
153
|
+
expect(() => service.endRun('non-existent')).toThrow('Run not found');
|
|
154
|
+
});
|
|
155
|
+
it('should calculate duration', () => {
|
|
156
|
+
const exp = service.createExperiment('test-exp');
|
|
157
|
+
const run = service.startRun(exp.id);
|
|
158
|
+
const endedRun = service.endRun(run.id);
|
|
159
|
+
expect(endedRun.durationMs).toBeGreaterThanOrEqual(0);
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
describe('getRun', () => {
|
|
163
|
+
it('should return run by id', () => {
|
|
164
|
+
const exp = service.createExperiment('test-exp');
|
|
165
|
+
const run = service.startRun(exp.id);
|
|
166
|
+
const retrieved = service.getRun(run.id);
|
|
167
|
+
expect(retrieved).toEqual(run);
|
|
168
|
+
});
|
|
169
|
+
it('should return undefined for non-existent run', () => {
|
|
170
|
+
expect(service.getRun('non-existent')).toBeUndefined();
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
describe('listRuns', () => {
|
|
174
|
+
it('should return empty array when no runs', () => {
|
|
175
|
+
expect(service.listRuns()).toEqual([]);
|
|
176
|
+
});
|
|
177
|
+
it('should list all runs', () => {
|
|
178
|
+
const exp = service.createExperiment('test-exp');
|
|
179
|
+
const run1 = service.startRun(exp.id);
|
|
180
|
+
const run2 = service.startRun(exp.id);
|
|
181
|
+
const runs = service.listRuns();
|
|
182
|
+
expect(runs).toHaveLength(2);
|
|
183
|
+
expect(runs).toContainEqual(run1);
|
|
184
|
+
expect(runs).toContainEqual(run2);
|
|
185
|
+
});
|
|
186
|
+
it('should filter runs by experiment id', () => {
|
|
187
|
+
const exp1 = service.createExperiment('exp1');
|
|
188
|
+
const exp2 = service.createExperiment('exp2');
|
|
189
|
+
const run1 = service.startRun(exp1.id);
|
|
190
|
+
const _run2 = service.startRun(exp2.id);
|
|
191
|
+
const runs = service.listRuns(exp1.id);
|
|
192
|
+
expect(runs).toHaveLength(1);
|
|
193
|
+
expect(runs[0]).toEqual(run1);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
describe('deleteRun', () => {
|
|
197
|
+
it('should delete run', () => {
|
|
198
|
+
const exp = service.createExperiment('test-exp');
|
|
199
|
+
const run = service.startRun(exp.id);
|
|
200
|
+
const deleted = service.deleteRun(run.id);
|
|
201
|
+
expect(deleted).toBe(true);
|
|
202
|
+
expect(service.getRun(run.id)).toBeUndefined();
|
|
203
|
+
});
|
|
204
|
+
it('should return false for non-existent run', () => {
|
|
205
|
+
expect(service.deleteRun('non-existent')).toBe(false);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
describe('logMetric', () => {
|
|
209
|
+
it('should log metric to run', () => {
|
|
210
|
+
const exp = service.createExperiment('test-exp');
|
|
211
|
+
const run = service.startRun(exp.id);
|
|
212
|
+
service.logMetric(run.id, 'accuracy', 0.95);
|
|
213
|
+
const updatedRun = service.getRun(run.id);
|
|
214
|
+
expect(updatedRun?.metrics.accuracy).toBe(0.95);
|
|
215
|
+
});
|
|
216
|
+
it('should throw error for non-existent run', () => {
|
|
217
|
+
expect(() => service.logMetric('non-existent', 'metric', 0.5)).toThrow('Run not found');
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
describe('logArtifact', () => {
|
|
221
|
+
it('should log artifact to run', () => {
|
|
222
|
+
const exp = service.createExperiment('test-exp');
|
|
223
|
+
const run = service.startRun(exp.id);
|
|
224
|
+
const artifact = service.logArtifact(run.id, 'trained-model', 'model', 'model content');
|
|
225
|
+
expect(artifact.name).toBe('trained-model');
|
|
226
|
+
expect(artifact.type).toBe('model');
|
|
227
|
+
const updatedRun = service.getRun(run.id);
|
|
228
|
+
expect(updatedRun?.artifacts).toHaveLength(1);
|
|
229
|
+
expect(updatedRun?.artifacts[0].name).toBe('trained-model');
|
|
230
|
+
expect(updatedRun?.artifacts[0].type).toBe('model');
|
|
231
|
+
});
|
|
232
|
+
it('should throw error for non-existent run', () => {
|
|
233
|
+
expect(() => service.logArtifact('non-existent', 'name', 'model', 'content')).toThrow('Run not found');
|
|
234
|
+
});
|
|
235
|
+
it('should support different artifact types', () => {
|
|
236
|
+
const exp = service.createExperiment('test-exp');
|
|
237
|
+
const run = service.startRun(exp.id);
|
|
238
|
+
service.logArtifact(run.id, 'plot1', 'plot', 'plot data');
|
|
239
|
+
service.logArtifact(run.id, 'log1', 'log', 'log data');
|
|
240
|
+
service.logArtifact(run.id, 'data1', 'data', 'data content');
|
|
241
|
+
service.logArtifact(run.id, 'other1', 'other', 'other content');
|
|
242
|
+
const updatedRun = service.getRun(run.id);
|
|
243
|
+
expect(updatedRun?.artifacts).toHaveLength(4);
|
|
244
|
+
});
|
|
245
|
+
});
|
|
246
|
+
describe('compareRuns', () => {
|
|
247
|
+
it('should compare two completed runs', () => {
|
|
248
|
+
const exp = service.createExperiment('test-exp');
|
|
249
|
+
const run1 = service.startRun(exp.id, { params: { lr: 0.001 } });
|
|
250
|
+
const run2 = service.startRun(exp.id, { params: { lr: 0.01 } });
|
|
251
|
+
service.logMetric(run1.id, 'accuracy', 0.9);
|
|
252
|
+
service.logMetric(run2.id, 'accuracy', 0.95);
|
|
253
|
+
service.endRun(run1.id, 'completed');
|
|
254
|
+
service.endRun(run2.id, 'completed');
|
|
255
|
+
const comparisons = service.compareRuns([run1.id, run2.id]);
|
|
256
|
+
expect(comparisons).toHaveLength(2);
|
|
257
|
+
expect(comparisons[0].runId).toBe(run1.id);
|
|
258
|
+
expect(comparisons[0].params).toEqual({ lr: 0.001 });
|
|
259
|
+
expect(comparisons[0].metrics.accuracy).toBe(0.9);
|
|
260
|
+
expect(comparisons[1].runId).toBe(run2.id);
|
|
261
|
+
});
|
|
262
|
+
it('should handle empty run list', () => {
|
|
263
|
+
const comparisons = service.compareRuns([]);
|
|
264
|
+
expect(comparisons).toEqual([]);
|
|
265
|
+
});
|
|
266
|
+
it('should filter out non-completed runs', () => {
|
|
267
|
+
const exp = service.createExperiment('test-exp');
|
|
268
|
+
const run1 = service.startRun(exp.id);
|
|
269
|
+
const run2 = service.startRun(exp.id);
|
|
270
|
+
service.endRun(run1.id, 'completed');
|
|
271
|
+
// run2 is still running
|
|
272
|
+
const comparisons = service.compareRuns([run1.id, run2.id]);
|
|
273
|
+
expect(comparisons).toHaveLength(1);
|
|
274
|
+
expect(comparisons[0].runId).toBe(run1.id);
|
|
275
|
+
});
|
|
276
|
+
});
|
|
277
|
+
describe('getBestRun', () => {
|
|
278
|
+
it('should return run with highest metric value', () => {
|
|
279
|
+
const exp = service.createExperiment('test-exp');
|
|
280
|
+
const run1 = service.startRun(exp.id);
|
|
281
|
+
const run2 = service.startRun(exp.id);
|
|
282
|
+
service.logMetric(run1.id, 'accuracy', 0.9);
|
|
283
|
+
service.logMetric(run2.id, 'accuracy', 0.95);
|
|
284
|
+
const best = service.getBestRun(exp.id, 'accuracy', 'max');
|
|
285
|
+
expect(best?.id).toBe(run2.id);
|
|
286
|
+
});
|
|
287
|
+
it('should return run with lowest metric value', () => {
|
|
288
|
+
const exp = service.createExperiment('test-exp');
|
|
289
|
+
const run1 = service.startRun(exp.id);
|
|
290
|
+
const run2 = service.startRun(exp.id);
|
|
291
|
+
service.logMetric(run1.id, 'loss', 0.5);
|
|
292
|
+
service.logMetric(run2.id, 'loss', 0.3);
|
|
293
|
+
const best = service.getBestRun(exp.id, 'loss', 'min');
|
|
294
|
+
expect(best?.id).toBe(run2.id);
|
|
295
|
+
});
|
|
296
|
+
it('should return undefined when no runs have the metric', () => {
|
|
297
|
+
const exp = service.createExperiment('test-exp');
|
|
298
|
+
service.startRun(exp.id);
|
|
299
|
+
const best = service.getBestRun(exp.id, 'accuracy', 'max');
|
|
300
|
+
expect(best).toBeUndefined();
|
|
301
|
+
});
|
|
302
|
+
it('should return undefined for experiment with no runs', () => {
|
|
303
|
+
const exp = service.createExperiment('test-exp');
|
|
304
|
+
const best = service.getBestRun(exp.id, 'accuracy', 'max');
|
|
305
|
+
expect(best).toBeUndefined();
|
|
306
|
+
});
|
|
307
|
+
});
|
|
308
|
+
describe('logMetrics', () => {
|
|
309
|
+
it('should log multiple metrics at once', () => {
|
|
310
|
+
const exp = service.createExperiment('test-exp');
|
|
311
|
+
const run = service.startRun(exp.id);
|
|
312
|
+
service.logMetrics(run.id, {
|
|
313
|
+
accuracy: 0.95,
|
|
314
|
+
precision: 0.92,
|
|
315
|
+
recall: 0.88,
|
|
316
|
+
});
|
|
317
|
+
const updatedRun = service.getRun(run.id);
|
|
318
|
+
expect(updatedRun?.metrics.accuracy).toBe(0.95);
|
|
319
|
+
expect(updatedRun?.metrics.precision).toBe(0.92);
|
|
320
|
+
expect(updatedRun?.metrics.recall).toBe(0.88);
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
describe('file storage', () => {
|
|
324
|
+
it('should work with file storage configuration', () => {
|
|
325
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
326
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
327
|
+
}
|
|
328
|
+
service.configure({
|
|
329
|
+
storage: 'file',
|
|
330
|
+
file: { directory: testStorageDir },
|
|
331
|
+
});
|
|
332
|
+
const exp = service.createExperiment('file-test');
|
|
333
|
+
const run = service.startRun(exp.id);
|
|
334
|
+
service.logMetric(run.id, 'accuracy', 0.95);
|
|
335
|
+
service.endRun(run.id);
|
|
336
|
+
// Verify we can retrieve from file storage
|
|
337
|
+
const retrieved = service.getExperiment(exp.id);
|
|
338
|
+
expect(retrieved?.name).toBe('file-test');
|
|
339
|
+
const retrievedRun = service.getRun(run.id);
|
|
340
|
+
expect(retrievedRun?.metrics.accuracy).toBe(0.95);
|
|
341
|
+
});
|
|
342
|
+
it('should list experiments from file storage', () => {
|
|
343
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
344
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
345
|
+
}
|
|
346
|
+
service.configure({
|
|
347
|
+
storage: 'file',
|
|
348
|
+
file: { directory: testStorageDir },
|
|
349
|
+
});
|
|
350
|
+
service.createExperiment('exp1');
|
|
351
|
+
service.createExperiment('exp2');
|
|
352
|
+
const experiments = service.listExperiments();
|
|
353
|
+
expect(experiments.length).toBeGreaterThanOrEqual(2);
|
|
354
|
+
});
|
|
355
|
+
it('should list runs from file storage', () => {
|
|
356
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
357
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
358
|
+
}
|
|
359
|
+
service.configure({
|
|
360
|
+
storage: 'file',
|
|
361
|
+
file: { directory: testStorageDir },
|
|
362
|
+
});
|
|
363
|
+
const exp = service.createExperiment('run-test');
|
|
364
|
+
service.startRun(exp.id);
|
|
365
|
+
service.startRun(exp.id);
|
|
366
|
+
const runs = service.listRuns(exp.id);
|
|
367
|
+
expect(runs.length).toBeGreaterThanOrEqual(2);
|
|
368
|
+
});
|
|
369
|
+
it('should delete experiment with file storage', () => {
|
|
370
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
371
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
372
|
+
}
|
|
373
|
+
service.configure({
|
|
374
|
+
storage: 'file',
|
|
375
|
+
file: { directory: testStorageDir },
|
|
376
|
+
});
|
|
377
|
+
const exp = service.createExperiment('delete-test');
|
|
378
|
+
const run = service.startRun(exp.id);
|
|
379
|
+
service.deleteExperiment(exp.id);
|
|
380
|
+
expect(service.getExperiment(exp.id)).toBeUndefined();
|
|
381
|
+
expect(service.getRun(run.id)).toBeUndefined();
|
|
382
|
+
});
|
|
383
|
+
it('should delete run with file storage', () => {
|
|
384
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
385
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
386
|
+
}
|
|
387
|
+
service.configure({
|
|
388
|
+
storage: 'file',
|
|
389
|
+
file: { directory: testStorageDir },
|
|
390
|
+
});
|
|
391
|
+
const exp = service.createExperiment('run-delete-test');
|
|
392
|
+
const run = service.startRun(exp.id);
|
|
393
|
+
service.deleteRun(run.id);
|
|
394
|
+
expect(service.getRun(run.id)).toBeUndefined();
|
|
395
|
+
});
|
|
396
|
+
it('should log artifact with file storage', () => {
|
|
397
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
398
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
399
|
+
}
|
|
400
|
+
service.configure({
|
|
401
|
+
storage: 'file',
|
|
402
|
+
file: { directory: testStorageDir },
|
|
403
|
+
});
|
|
404
|
+
const exp = service.createExperiment('artifact-test');
|
|
405
|
+
const run = service.startRun(exp.id);
|
|
406
|
+
const artifact = service.logArtifact(run.id, 'model', 'model', 'model data');
|
|
407
|
+
expect(artifact.path).toBeDefined();
|
|
408
|
+
expect(artifact.size).toBeGreaterThan(0);
|
|
409
|
+
});
|
|
410
|
+
it('should handle non-existent experiment in file storage', () => {
|
|
411
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
412
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
413
|
+
}
|
|
414
|
+
service.configure({
|
|
415
|
+
storage: 'file',
|
|
416
|
+
file: { directory: testStorageDir },
|
|
417
|
+
});
|
|
418
|
+
expect(service.getExperiment('non-existent')).toBeUndefined();
|
|
419
|
+
});
|
|
420
|
+
it('should handle non-existent run in file storage', () => {
|
|
421
|
+
if (!(0, fs_1.existsSync)(testStorageDir)) {
|
|
422
|
+
(0, fs_1.mkdirSync)(testStorageDir, { recursive: true });
|
|
423
|
+
}
|
|
424
|
+
service.configure({
|
|
425
|
+
storage: 'file',
|
|
426
|
+
file: { directory: testStorageDir },
|
|
427
|
+
});
|
|
428
|
+
expect(service.getRun('non-existent')).toBeUndefined();
|
|
429
|
+
});
|
|
430
|
+
it('should return empty array when no experiments in file storage', () => {
|
|
431
|
+
const emptyDir = (0, path_1.join)(testStorageDir, 'empty');
|
|
432
|
+
if (!(0, fs_1.existsSync)(emptyDir)) {
|
|
433
|
+
(0, fs_1.mkdirSync)(emptyDir, { recursive: true });
|
|
434
|
+
}
|
|
435
|
+
service.configure({
|
|
436
|
+
storage: 'file',
|
|
437
|
+
file: { directory: emptyDir },
|
|
438
|
+
});
|
|
439
|
+
expect(service.listExperiments()).toEqual([]);
|
|
440
|
+
});
|
|
441
|
+
});
|
|
442
|
+
describe('getArtifact', () => {
|
|
443
|
+
it('should get artifact by id', () => {
|
|
444
|
+
const exp = service.createExperiment('test-exp');
|
|
445
|
+
const run = service.startRun(exp.id);
|
|
446
|
+
const artifact = service.logArtifact(run.id, 'model', 'model', 'data');
|
|
447
|
+
const retrieved = service.getArtifact(run.id, artifact.id);
|
|
448
|
+
expect(retrieved).toBeDefined();
|
|
449
|
+
expect(retrieved?.id).toBe(artifact.id);
|
|
450
|
+
});
|
|
451
|
+
it('should return undefined for non-existent artifact', () => {
|
|
452
|
+
const exp = service.createExperiment('test-exp');
|
|
453
|
+
const run = service.startRun(exp.id);
|
|
454
|
+
expect(service.getArtifact(run.id, 'non-existent')).toBeUndefined();
|
|
455
|
+
});
|
|
456
|
+
it('should return undefined for non-existent run', () => {
|
|
457
|
+
expect(service.getArtifact('non-existent', 'artifact-id')).toBeUndefined();
|
|
458
|
+
});
|
|
459
|
+
});
|
|
460
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @Experiment decorator - Auto-track training runs
|
|
3
|
+
*/
|
|
4
|
+
import 'reflect-metadata';
|
|
5
|
+
export declare const EXPERIMENT_METADATA_KEY: unique symbol;
|
|
6
|
+
export interface ExperimentOptions {
|
|
7
|
+
name?: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
tags?: string[];
|
|
10
|
+
autoLogParams?: boolean;
|
|
11
|
+
autoLogMetrics?: boolean;
|
|
12
|
+
}
|
|
13
|
+
export interface ExperimentMetadata {
|
|
14
|
+
name: string;
|
|
15
|
+
description?: string;
|
|
16
|
+
tags?: string[];
|
|
17
|
+
autoLogParams: boolean;
|
|
18
|
+
autoLogMetrics: boolean;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Mark a class as an ML experiment for auto-tracking.
|
|
22
|
+
* When combined with @Train, training runs are automatically logged.
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* @Experiment({
|
|
27
|
+
* name: 'sentiment-classifier',
|
|
28
|
+
* description: 'Training sentiment classification models',
|
|
29
|
+
* tags: ['nlp', 'classification']
|
|
30
|
+
* })
|
|
31
|
+
* @Model({ name: 'sentiment', version: '1.0.0', framework: 'custom' })
|
|
32
|
+
* @Injectable()
|
|
33
|
+
* class SentimentClassifier {
|
|
34
|
+
* @Train()
|
|
35
|
+
* async train(data: TrainingData) {
|
|
36
|
+
* // This run will be automatically tracked
|
|
37
|
+
* }
|
|
38
|
+
* }
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
export declare function Experiment(options?: ExperimentOptions): ClassDecorator;
|
|
42
|
+
export declare function getExperimentMetadata(target: object): ExperimentMetadata | undefined;
|
|
43
|
+
export declare function hasExperimentMetadata(target: object): boolean;
|
|
44
|
+
//# sourceMappingURL=experiment.decorator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment.decorator.d.ts","sourceRoot":"","sources":["../../src/experiments/experiment.decorator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,kBAAkB,CAAC;AAE1B,eAAO,MAAM,uBAAuB,eAAsC,CAAC;AAE3E,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,UAAU,CAAC,OAAO,GAAE,iBAAsB,GAAG,cAAc,CAa1E;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CAEpF;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE7D"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @Experiment decorator - Auto-track training runs
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.EXPERIMENT_METADATA_KEY = void 0;
|
|
7
|
+
exports.Experiment = Experiment;
|
|
8
|
+
exports.getExperimentMetadata = getExperimentMetadata;
|
|
9
|
+
exports.hasExperimentMetadata = hasExperimentMetadata;
|
|
10
|
+
require("reflect-metadata");
|
|
11
|
+
exports.EXPERIMENT_METADATA_KEY = Symbol('hazel:experiment:metadata');
|
|
12
|
+
/**
|
|
13
|
+
* Mark a class as an ML experiment for auto-tracking.
|
|
14
|
+
* When combined with @Train, training runs are automatically logged.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* @Experiment({
|
|
19
|
+
* name: 'sentiment-classifier',
|
|
20
|
+
* description: 'Training sentiment classification models',
|
|
21
|
+
* tags: ['nlp', 'classification']
|
|
22
|
+
* })
|
|
23
|
+
* @Model({ name: 'sentiment', version: '1.0.0', framework: 'custom' })
|
|
24
|
+
* @Injectable()
|
|
25
|
+
* class SentimentClassifier {
|
|
26
|
+
* @Train()
|
|
27
|
+
* async train(data: TrainingData) {
|
|
28
|
+
* // This run will be automatically tracked
|
|
29
|
+
* }
|
|
30
|
+
* }
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
function Experiment(options = {}) {
|
|
34
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
35
|
+
return function (target) {
|
|
36
|
+
const metadata = {
|
|
37
|
+
name: options.name ?? target.name,
|
|
38
|
+
description: options.description,
|
|
39
|
+
tags: options.tags,
|
|
40
|
+
autoLogParams: options.autoLogParams ?? true,
|
|
41
|
+
autoLogMetrics: options.autoLogMetrics ?? true,
|
|
42
|
+
};
|
|
43
|
+
Reflect.defineMetadata(exports.EXPERIMENT_METADATA_KEY, metadata, target);
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function getExperimentMetadata(target) {
|
|
47
|
+
return Reflect.getMetadata(exports.EXPERIMENT_METADATA_KEY, target);
|
|
48
|
+
}
|
|
49
|
+
function hasExperimentMetadata(target) {
|
|
50
|
+
return Reflect.hasMetadata(exports.EXPERIMENT_METADATA_KEY, target);
|
|
51
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Experiment Service - ML experiment tracking and management
|
|
3
|
+
*/
|
|
4
|
+
import type { Experiment, Run, Artifact, ExperimentConfig, RunComparison } from './experiment.types';
|
|
5
|
+
export declare class ExperimentService {
|
|
6
|
+
private experiments;
|
|
7
|
+
private runs;
|
|
8
|
+
private config;
|
|
9
|
+
private storageDir;
|
|
10
|
+
configure(config: ExperimentConfig): void;
|
|
11
|
+
createExperiment(name: string, options?: {
|
|
12
|
+
description?: string;
|
|
13
|
+
tags?: string[];
|
|
14
|
+
}): Experiment;
|
|
15
|
+
getExperiment(id: string): Experiment | undefined;
|
|
16
|
+
listExperiments(): Experiment[];
|
|
17
|
+
deleteExperiment(id: string): boolean;
|
|
18
|
+
startRun(experimentId: string, options?: {
|
|
19
|
+
name?: string;
|
|
20
|
+
params?: Record<string, unknown>;
|
|
21
|
+
tags?: string[];
|
|
22
|
+
}): Run;
|
|
23
|
+
endRun(runId: string, status?: 'completed' | 'failed' | 'aborted'): Run;
|
|
24
|
+
getRun(id: string): Run | undefined;
|
|
25
|
+
listRuns(experimentId?: string): Run[];
|
|
26
|
+
deleteRun(id: string): boolean;
|
|
27
|
+
logMetric(runId: string, key: string, value: number): void;
|
|
28
|
+
logMetrics(runId: string, metrics: Record<string, number>): void;
|
|
29
|
+
getBestRun(experimentId: string, metric: string, mode?: 'min' | 'max'): Run | undefined;
|
|
30
|
+
logArtifact(runId: string, name: string, type: Artifact['type'], content: string | Buffer, metadata?: Record<string, unknown>): Artifact;
|
|
31
|
+
getArtifact(runId: string, artifactId: string): Artifact | undefined;
|
|
32
|
+
compareRuns(runIds: string[]): RunComparison[];
|
|
33
|
+
private saveExperimentToFile;
|
|
34
|
+
private loadExperimentFromFile;
|
|
35
|
+
private loadAllExperimentsFromFile;
|
|
36
|
+
private saveRunToFile;
|
|
37
|
+
private loadRunFromFile;
|
|
38
|
+
private loadAllRunsFromFile;
|
|
39
|
+
private deleteArtifactFile;
|
|
40
|
+
private getArtifactExtension;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=experiment.service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment.service.d.ts","sourceRoot":"","sources":["../../src/experiments/experiment.service.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EACV,GAAG,EACH,QAAQ,EACR,gBAAgB,EAChB,aAAa,EACd,MAAM,oBAAoB,CAAC;AAM5B,qBACa,iBAAiB;IAC5B,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,IAAI,CAA+B;IAC3C,OAAO,CAAC,MAAM,CAA2C;IACzD,OAAO,CAAC,UAAU,CAA2B;IAE7C,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAUzC,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;KAAO,GACtD,UAAU;IAoBb,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAOjD,eAAe,IAAI,UAAU,EAAE;IAO/B,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IA0BrC,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;KAAO,GACjF,GAAG;IAuBN,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,WAAW,GAAG,QAAQ,GAAG,SAAuB,GAAG,GAAG;IAkBpF,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS;IAOnC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE;IAgBtC,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAuB9B,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAe1D,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAMhE,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,KAAK,GAAG,KAAa,GAAG,GAAG,GAAG,SAAS;IAkB9F,WAAW,CACT,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,MAAM,GAAG,MAAM,EACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACjC,QAAQ;IAiDX,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IASpE,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE;IAc9C,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,sBAAsB;IAW9B,OAAO,CAAC,0BAA0B;IAgBlC,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,eAAe;IAuBvB,OAAO,CAAC,mBAAmB;IAuB3B,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,oBAAoB;CAc7B"}
|