@hazeljs/data 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +192 -0
- package/README.md +308 -0
- package/dist/connectors/connector.interface.d.ts +29 -0
- package/dist/connectors/connector.interface.d.ts.map +1 -0
- package/dist/connectors/connector.interface.js +6 -0
- package/dist/connectors/csv.connector.d.ts +63 -0
- package/dist/connectors/csv.connector.d.ts.map +1 -0
- package/dist/connectors/csv.connector.js +147 -0
- package/dist/connectors/http.connector.d.ts +68 -0
- package/dist/connectors/http.connector.d.ts.map +1 -0
- package/dist/connectors/http.connector.js +131 -0
- package/dist/connectors/index.d.ts +7 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +12 -0
- package/dist/connectors/memory.connector.d.ts +38 -0
- package/dist/connectors/memory.connector.d.ts.map +1 -0
- package/dist/connectors/memory.connector.js +56 -0
- package/dist/connectors/memory.connector.test.d.ts +2 -0
- package/dist/connectors/memory.connector.test.d.ts.map +1 -0
- package/dist/connectors/memory.connector.test.js +43 -0
- package/dist/data.module.d.ts +30 -0
- package/dist/data.module.d.ts.map +1 -0
- package/dist/data.module.js +120 -0
- package/dist/data.module.test.d.ts +2 -0
- package/dist/data.module.test.d.ts.map +1 -0
- package/dist/data.module.test.js +28 -0
- package/dist/data.types.d.ts +67 -0
- package/dist/data.types.d.ts.map +1 -0
- package/dist/data.types.js +5 -0
- package/dist/decorators/index.d.ts +6 -0
- package/dist/decorators/index.d.ts.map +1 -0
- package/dist/decorators/index.js +24 -0
- package/dist/decorators/pii.decorator.d.ts +59 -0
- package/dist/decorators/pii.decorator.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.js +197 -0
- package/dist/decorators/pii.decorator.test.d.ts +2 -0
- package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.test.js +150 -0
- package/dist/decorators/pipeline.decorator.d.ts +22 -0
- package/dist/decorators/pipeline.decorator.d.ts.map +1 -0
- package/dist/decorators/pipeline.decorator.js +42 -0
- package/dist/decorators/pipeline.decorator.test.d.ts +2 -0
- package/dist/decorators/pipeline.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pipeline.decorator.test.js +104 -0
- package/dist/decorators/stream.decorator.d.ts +31 -0
- package/dist/decorators/stream.decorator.d.ts.map +1 -0
- package/dist/decorators/stream.decorator.js +48 -0
- package/dist/decorators/transform.decorator.d.ts +29 -0
- package/dist/decorators/transform.decorator.d.ts.map +1 -0
- package/dist/decorators/transform.decorator.js +41 -0
- package/dist/decorators/validate.decorator.d.ts +34 -0
- package/dist/decorators/validate.decorator.d.ts.map +1 -0
- package/dist/decorators/validate.decorator.js +49 -0
- package/dist/flink.service.d.ts +80 -0
- package/dist/flink.service.d.ts.map +1 -0
- package/dist/flink.service.js +134 -0
- package/dist/flink.service.test.d.ts +2 -0
- package/dist/flink.service.test.d.ts.map +1 -0
- package/dist/flink.service.test.js +60 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +96 -0
- package/dist/pipelines/etl.service.d.ts +59 -0
- package/dist/pipelines/etl.service.d.ts.map +1 -0
- package/dist/pipelines/etl.service.js +223 -0
- package/dist/pipelines/etl.service.test.d.ts +2 -0
- package/dist/pipelines/etl.service.test.d.ts.map +1 -0
- package/dist/pipelines/etl.service.test.js +319 -0
- package/dist/pipelines/pipeline.base.d.ts +24 -0
- package/dist/pipelines/pipeline.base.d.ts.map +1 -0
- package/dist/pipelines/pipeline.base.js +29 -0
- package/dist/pipelines/pipeline.base.test.d.ts +2 -0
- package/dist/pipelines/pipeline.base.test.d.ts.map +1 -0
- package/dist/pipelines/pipeline.base.test.js +38 -0
- package/dist/pipelines/pipeline.builder.d.ts +95 -0
- package/dist/pipelines/pipeline.builder.d.ts.map +1 -0
- package/dist/pipelines/pipeline.builder.js +212 -0
- package/dist/pipelines/pipeline.builder.test.d.ts +2 -0
- package/dist/pipelines/pipeline.builder.test.d.ts.map +1 -0
- package/dist/pipelines/pipeline.builder.test.js +185 -0
- package/dist/pipelines/stream.service.d.ts +12 -0
- package/dist/pipelines/stream.service.d.ts.map +1 -0
- package/dist/pipelines/stream.service.js +58 -0
- package/dist/pipelines/stream.service.test.d.ts +2 -0
- package/dist/pipelines/stream.service.test.d.ts.map +1 -0
- package/dist/pipelines/stream.service.test.js +103 -0
- package/dist/quality/quality.service.d.ts +87 -0
- package/dist/quality/quality.service.d.ts.map +1 -0
- package/dist/quality/quality.service.js +326 -0
- package/dist/quality/quality.service.test.d.ts +2 -0
- package/dist/quality/quality.service.test.d.ts.map +1 -0
- package/dist/quality/quality.service.test.js +128 -0
- package/dist/schema/schema.d.ts +127 -0
- package/dist/schema/schema.d.ts.map +1 -0
- package/dist/schema/schema.js +487 -0
- package/dist/schema/schema.test.d.ts +2 -0
- package/dist/schema/schema.test.d.ts.map +1 -0
- package/dist/schema/schema.test.js +411 -0
- package/dist/streaming/flink/flink.client.d.ts +96 -0
- package/dist/streaming/flink/flink.client.d.ts.map +1 -0
- package/dist/streaming/flink/flink.client.js +267 -0
- package/dist/streaming/flink/flink.client.test.d.ts +2 -0
- package/dist/streaming/flink/flink.client.test.d.ts.map +1 -0
- package/dist/streaming/flink/flink.client.test.js +59 -0
- package/dist/streaming/flink/flink.job.d.ts +29 -0
- package/dist/streaming/flink/flink.job.d.ts.map +1 -0
- package/dist/streaming/flink/flink.job.js +27 -0
- package/dist/streaming/flink/flink.job.test.d.ts +2 -0
- package/dist/streaming/flink/flink.job.test.d.ts.map +1 -0
- package/dist/streaming/flink/flink.job.test.js +37 -0
- package/dist/streaming/flink/flink.operators.d.ts +35 -0
- package/dist/streaming/flink/flink.operators.d.ts.map +1 -0
- package/dist/streaming/flink/flink.operators.js +43 -0
- package/dist/streaming/flink/flink.operators.test.d.ts +2 -0
- package/dist/streaming/flink/flink.operators.test.d.ts.map +1 -0
- package/dist/streaming/flink/flink.operators.test.js +38 -0
- package/dist/streaming/stream.builder.d.ts +22 -0
- package/dist/streaming/stream.builder.d.ts.map +1 -0
- package/dist/streaming/stream.builder.js +50 -0
- package/dist/streaming/stream.builder.test.d.ts +2 -0
- package/dist/streaming/stream.builder.test.d.ts.map +1 -0
- package/dist/streaming/stream.builder.test.js +59 -0
- package/dist/streaming/stream.processor.d.ts +66 -0
- package/dist/streaming/stream.processor.d.ts.map +1 -0
- package/dist/streaming/stream.processor.js +178 -0
- package/dist/streaming/stream.processor.test.d.ts +2 -0
- package/dist/streaming/stream.processor.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.test.js +151 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.windowing.test.js +69 -0
- package/dist/telemetry/telemetry.d.ts +124 -0
- package/dist/telemetry/telemetry.d.ts.map +1 -0
- package/dist/telemetry/telemetry.js +259 -0
- package/dist/telemetry/telemetry.test.d.ts +2 -0
- package/dist/telemetry/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry/telemetry.test.js +51 -0
- package/dist/testing/index.d.ts +12 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/pipeline-test-harness.d.ts +40 -0
- package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.js +55 -0
- package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
- package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.test.js +102 -0
- package/dist/testing/schema-faker.d.ts +32 -0
- package/dist/testing/schema-faker.d.ts.map +1 -0
- package/dist/testing/schema-faker.js +91 -0
- package/dist/testing/schema-faker.test.d.ts +2 -0
- package/dist/testing/schema-faker.test.d.ts.map +1 -0
- package/dist/testing/schema-faker.test.js +66 -0
- package/dist/transformers/built-in.transformers.d.ts +12 -0
- package/dist/transformers/built-in.transformers.d.ts.map +1 -0
- package/dist/transformers/built-in.transformers.js +75 -0
- package/dist/transformers/built-in.transformers.test.d.ts +2 -0
- package/dist/transformers/built-in.transformers.test.d.ts.map +1 -0
- package/dist/transformers/built-in.transformers.test.js +85 -0
- package/dist/transformers/transformer.service.d.ts +14 -0
- package/dist/transformers/transformer.service.d.ts.map +1 -0
- package/dist/transformers/transformer.service.js +65 -0
- package/dist/transformers/transformer.service.test.d.ts +2 -0
- package/dist/transformers/transformer.service.test.d.ts.map +1 -0
- package/dist/transformers/transformer.service.test.js +42 -0
- package/dist/validators/schema.validator.d.ts +21 -0
- package/dist/validators/schema.validator.d.ts.map +1 -0
- package/dist/validators/schema.validator.js +40 -0
- package/dist/validators/schema.validator.test.d.ts +2 -0
- package/dist/validators/schema.validator.test.d.ts.map +1 -0
- package/dist/validators/schema.validator.test.js +42 -0
- package/package.json +53 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
const etl_service_1 = require("./etl.service");
|
|
13
|
+
const schema_validator_1 = require("../validators/schema.validator");
|
|
14
|
+
const decorators_1 = require("../decorators");
|
|
15
|
+
const schema_1 = require("../schema/schema");
|
|
16
|
+
let TestPipeline = class TestPipeline {
|
|
17
|
+
double(data) {
|
|
18
|
+
const n = data.x;
|
|
19
|
+
return { x: n * 2 };
|
|
20
|
+
}
|
|
21
|
+
validate(data) {
|
|
22
|
+
return data;
|
|
23
|
+
}
|
|
24
|
+
add(data) {
|
|
25
|
+
return { ...data, y: 10 };
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
__decorate([
|
|
29
|
+
(0, decorators_1.Transform)({ step: 1, name: 'double' }),
|
|
30
|
+
__metadata("design:type", Function),
|
|
31
|
+
__metadata("design:paramtypes", [Object]),
|
|
32
|
+
__metadata("design:returntype", void 0)
|
|
33
|
+
], TestPipeline.prototype, "double", null);
|
|
34
|
+
__decorate([
|
|
35
|
+
(0, decorators_1.Validate)({
|
|
36
|
+
step: 2,
|
|
37
|
+
name: 'validate',
|
|
38
|
+
schema: schema_1.Schema.object({ x: schema_1.Schema.number().min(0) }),
|
|
39
|
+
}),
|
|
40
|
+
__metadata("design:type", Function),
|
|
41
|
+
__metadata("design:paramtypes", [Object]),
|
|
42
|
+
__metadata("design:returntype", void 0)
|
|
43
|
+
], TestPipeline.prototype, "validate", null);
|
|
44
|
+
__decorate([
|
|
45
|
+
(0, decorators_1.Transform)({ step: 3, name: 'add' }),
|
|
46
|
+
__metadata("design:type", Function),
|
|
47
|
+
__metadata("design:paramtypes", [Object]),
|
|
48
|
+
__metadata("design:returntype", void 0)
|
|
49
|
+
], TestPipeline.prototype, "add", null);
|
|
50
|
+
TestPipeline = __decorate([
|
|
51
|
+
(0, decorators_1.Pipeline)('test-pipeline')
|
|
52
|
+
], TestPipeline);
|
|
53
|
+
let TransformOnlyPipeline = class TransformOnlyPipeline {
|
|
54
|
+
a(data) {
|
|
55
|
+
return { ...data, a: 1 };
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
__decorate([
|
|
59
|
+
(0, decorators_1.Transform)({ step: 1, name: 'a' }),
|
|
60
|
+
__metadata("design:type", Function),
|
|
61
|
+
__metadata("design:paramtypes", [Object]),
|
|
62
|
+
__metadata("design:returntype", void 0)
|
|
63
|
+
], TransformOnlyPipeline.prototype, "a", null);
|
|
64
|
+
TransformOnlyPipeline = __decorate([
|
|
65
|
+
(0, decorators_1.Pipeline)('transform-only')
|
|
66
|
+
], TransformOnlyPipeline);
|
|
67
|
+
let NoStepsPipeline = class NoStepsPipeline {
|
|
68
|
+
};
|
|
69
|
+
NoStepsPipeline = __decorate([
|
|
70
|
+
(0, decorators_1.Pipeline)('no-steps')
|
|
71
|
+
], NoStepsPipeline);
|
|
72
|
+
describe('ETLService', () => {
|
|
73
|
+
let etlService;
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
|
|
76
|
+
});
|
|
77
|
+
it('executes pipeline steps sequentially', async () => {
|
|
78
|
+
const pipeline = new TestPipeline();
|
|
79
|
+
const result = await etlService.execute(pipeline, { x: 5 });
|
|
80
|
+
expect(result).toEqual({ x: 10, y: 10 });
|
|
81
|
+
});
|
|
82
|
+
it('extractSteps returns sorted steps', () => {
|
|
83
|
+
const pipeline = new TestPipeline();
|
|
84
|
+
const steps = etlService.extractSteps(pipeline);
|
|
85
|
+
expect(steps).toHaveLength(3);
|
|
86
|
+
expect(steps[0].step).toBe(1);
|
|
87
|
+
expect(steps[1].step).toBe(2);
|
|
88
|
+
expect(steps[2].step).toBe(3);
|
|
89
|
+
});
|
|
90
|
+
it('throws when pipeline has no steps', async () => {
|
|
91
|
+
const pipeline = new NoStepsPipeline();
|
|
92
|
+
await expect(etlService.execute(pipeline, {})).rejects.toThrow('has no steps');
|
|
93
|
+
});
|
|
94
|
+
it('throws when step method not found on instance', async () => {
|
|
95
|
+
const pipeline = new TestPipeline();
|
|
96
|
+
pipeline.double = null;
|
|
97
|
+
await expect(etlService.execute(pipeline, { x: 5 })).rejects.toThrow('not found');
|
|
98
|
+
});
|
|
99
|
+
it('validates data at validate step', async () => {
|
|
100
|
+
const pipeline = new TestPipeline();
|
|
101
|
+
await expect(etlService.execute(pipeline, { x: -1 })).rejects.toThrow();
|
|
102
|
+
});
|
|
103
|
+
it('extractSteps returns only transform/validate steps', () => {
|
|
104
|
+
const pipeline = new TransformOnlyPipeline();
|
|
105
|
+
const steps = etlService.extractSteps(pipeline);
|
|
106
|
+
expect(steps).toHaveLength(1);
|
|
107
|
+
expect(steps[0].type).toBe('transform');
|
|
108
|
+
});
|
|
109
|
+
it('onStepComplete emits events for each step', async () => {
|
|
110
|
+
const events = [];
|
|
111
|
+
etlService.onStepComplete((e) => events.push({ step: e.step, stepName: e.stepName, success: e.success }));
|
|
112
|
+
const pipeline = new TestPipeline();
|
|
113
|
+
await etlService.execute(pipeline, { x: 5 });
|
|
114
|
+
expect(events).toHaveLength(3);
|
|
115
|
+
expect(events.every((e) => e.success)).toBe(true);
|
|
116
|
+
expect(events.map((e) => e.stepName)).toEqual(['double', 'validate', 'add']);
|
|
117
|
+
});
|
|
118
|
+
it('executeBatch returns results and errors', async () => {
|
|
119
|
+
const pipeline = new TestPipeline();
|
|
120
|
+
const { results, errors } = await etlService.executeBatch(pipeline, [{ x: 1 }, { x: -1 }, { x: 2 }], { concurrency: 2 });
|
|
121
|
+
expect(results).toHaveLength(2);
|
|
122
|
+
expect(results[0]).toEqual({ x: 2, y: 10 });
|
|
123
|
+
expect(results[1]).toEqual({ x: 4, y: 10 });
|
|
124
|
+
expect(errors).toHaveLength(1);
|
|
125
|
+
expect(errors[0].item).toEqual({ x: -1 });
|
|
126
|
+
});
|
|
127
|
+
it('executeBatch wraps non-Error rejections', async () => {
|
|
128
|
+
let ThrowStringPipeline = class ThrowStringPipeline {
|
|
129
|
+
fail() {
|
|
130
|
+
throw 'string rejection';
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
__decorate([
|
|
134
|
+
(0, decorators_1.Transform)({ step: 1, name: 'fail' }),
|
|
135
|
+
__metadata("design:type", Function),
|
|
136
|
+
__metadata("design:paramtypes", []),
|
|
137
|
+
__metadata("design:returntype", void 0)
|
|
138
|
+
], ThrowStringPipeline.prototype, "fail", null);
|
|
139
|
+
ThrowStringPipeline = __decorate([
|
|
140
|
+
(0, decorators_1.Pipeline)('throw-string')
|
|
141
|
+
], ThrowStringPipeline);
|
|
142
|
+
const { errors } = await etlService.executeBatch(new ThrowStringPipeline(), [{}]);
|
|
143
|
+
expect(errors).toHaveLength(1);
|
|
144
|
+
expect(errors[0].error).toBeInstanceOf(Error);
|
|
145
|
+
expect(errors[0].error.message).toContain('string rejection');
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
let ConditionalPipeline = class ConditionalPipeline {
|
|
149
|
+
always(data) {
|
|
150
|
+
return { ...data, step: 1 };
|
|
151
|
+
}
|
|
152
|
+
whenStep(data) {
|
|
153
|
+
return { ...data, step: 2 };
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
__decorate([
|
|
157
|
+
(0, decorators_1.Transform)({ step: 1, name: 'always' }),
|
|
158
|
+
__metadata("design:type", Function),
|
|
159
|
+
__metadata("design:paramtypes", [Object]),
|
|
160
|
+
__metadata("design:returntype", void 0)
|
|
161
|
+
], ConditionalPipeline.prototype, "always", null);
|
|
162
|
+
__decorate([
|
|
163
|
+
(0, decorators_1.Transform)({
|
|
164
|
+
step: 2,
|
|
165
|
+
name: 'when-true',
|
|
166
|
+
when: (d) => d.skip !== true,
|
|
167
|
+
}),
|
|
168
|
+
__metadata("design:type", Function),
|
|
169
|
+
__metadata("design:paramtypes", [Object]),
|
|
170
|
+
__metadata("design:returntype", void 0)
|
|
171
|
+
], ConditionalPipeline.prototype, "whenStep", null);
|
|
172
|
+
ConditionalPipeline = __decorate([
|
|
173
|
+
(0, decorators_1.Pipeline)('conditional-pipeline')
|
|
174
|
+
], ConditionalPipeline);
|
|
175
|
+
describe('ETLService conditional steps', () => {
|
|
176
|
+
let etlService;
|
|
177
|
+
beforeEach(() => {
|
|
178
|
+
etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
|
|
179
|
+
});
|
|
180
|
+
it('skips step when when predicate returns false', async () => {
|
|
181
|
+
const pipeline = new ConditionalPipeline();
|
|
182
|
+
const events = [];
|
|
183
|
+
etlService.onStepComplete((e) => events.push({ stepName: e.stepName, skipped: e.skipped }));
|
|
184
|
+
const result = await etlService.execute(pipeline, {
|
|
185
|
+
skip: true,
|
|
186
|
+
});
|
|
187
|
+
expect(result.step).toBe(1);
|
|
188
|
+
expect(events.find((e) => e.stepName === 'when-true')?.skipped).toBe(true);
|
|
189
|
+
});
|
|
190
|
+
it('runs step when when predicate returns true', async () => {
|
|
191
|
+
const pipeline = new ConditionalPipeline();
|
|
192
|
+
const result = await etlService.execute(pipeline, {});
|
|
193
|
+
expect(result.step).toBe(2);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
let RetryPipeline = class RetryPipeline {
|
|
197
|
+
constructor() {
|
|
198
|
+
this.attempts = 0;
|
|
199
|
+
}
|
|
200
|
+
retryStep(data) {
|
|
201
|
+
this.attempts++;
|
|
202
|
+
if (this.attempts < 2)
|
|
203
|
+
throw new Error('Fail');
|
|
204
|
+
return { ...data, ok: true };
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
__decorate([
|
|
208
|
+
(0, decorators_1.Transform)({ step: 1, name: 'retry', retry: { attempts: 3, delay: 10, backoff: 'exponential' } }),
|
|
209
|
+
__metadata("design:type", Function),
|
|
210
|
+
__metadata("design:paramtypes", [Object]),
|
|
211
|
+
__metadata("design:returntype", void 0)
|
|
212
|
+
], RetryPipeline.prototype, "retryStep", null);
|
|
213
|
+
RetryPipeline = __decorate([
|
|
214
|
+
(0, decorators_1.Pipeline)('retry-pipeline')
|
|
215
|
+
], RetryPipeline);
|
|
216
|
+
const dlqCaptured = [];
|
|
217
|
+
let DLQPipeline = class DLQPipeline {
|
|
218
|
+
failStep(data) {
|
|
219
|
+
if (data.skip)
|
|
220
|
+
return data;
|
|
221
|
+
throw new Error('DLQ test error');
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
__decorate([
|
|
225
|
+
(0, decorators_1.Transform)({
|
|
226
|
+
step: 1,
|
|
227
|
+
name: 'fail',
|
|
228
|
+
dlq: {
|
|
229
|
+
handler: (item, err) => {
|
|
230
|
+
dlqCaptured.push({ item, error: err.message });
|
|
231
|
+
},
|
|
232
|
+
},
|
|
233
|
+
}),
|
|
234
|
+
__metadata("design:type", Function),
|
|
235
|
+
__metadata("design:paramtypes", [Object]),
|
|
236
|
+
__metadata("design:returntype", void 0)
|
|
237
|
+
], DLQPipeline.prototype, "failStep", null);
|
|
238
|
+
DLQPipeline = __decorate([
|
|
239
|
+
(0, decorators_1.Pipeline)('dlq-pipeline')
|
|
240
|
+
], DLQPipeline);
|
|
241
|
+
describe('ETLService retry and DLQ', () => {
|
|
242
|
+
let etlService;
|
|
243
|
+
beforeEach(() => {
|
|
244
|
+
etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
|
|
245
|
+
dlqCaptured.length = 0;
|
|
246
|
+
});
|
|
247
|
+
it('retries step on failure', async () => {
|
|
248
|
+
const pipeline = new RetryPipeline();
|
|
249
|
+
const result = await etlService.execute(pipeline, {});
|
|
250
|
+
expect(result.ok).toBe(true);
|
|
251
|
+
expect(pipeline.attempts).toBe(2);
|
|
252
|
+
});
|
|
253
|
+
it('dlq receives failed item and pipeline continues with unchanged data', async () => {
|
|
254
|
+
const pipeline = new DLQPipeline();
|
|
255
|
+
const result = await etlService.execute(pipeline, { x: 1 });
|
|
256
|
+
expect(dlqCaptured).toHaveLength(1);
|
|
257
|
+
expect(dlqCaptured[0].error).toBe('DLQ test error');
|
|
258
|
+
expect(result).toEqual({ x: 1 });
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
let TimeoutPipeline = class TimeoutPipeline {
|
|
262
|
+
async slowStep() {
|
|
263
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
264
|
+
return { done: true };
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
__decorate([
|
|
268
|
+
(0, decorators_1.Transform)({ step: 1, name: 'slow', timeoutMs: 50 }),
|
|
269
|
+
__metadata("design:type", Function),
|
|
270
|
+
__metadata("design:paramtypes", []),
|
|
271
|
+
__metadata("design:returntype", Promise)
|
|
272
|
+
], TimeoutPipeline.prototype, "slowStep", null);
|
|
273
|
+
TimeoutPipeline = __decorate([
|
|
274
|
+
(0, decorators_1.Pipeline)('timeout-pipeline')
|
|
275
|
+
], TimeoutPipeline);
|
|
276
|
+
let RetryFixedPipeline = class RetryFixedPipeline {
|
|
277
|
+
constructor() {
|
|
278
|
+
this.attempts = 0;
|
|
279
|
+
}
|
|
280
|
+
retryStep() {
|
|
281
|
+
this.attempts++;
|
|
282
|
+
if (this.attempts < 2)
|
|
283
|
+
throw new Error('Fail');
|
|
284
|
+
return { ok: true };
|
|
285
|
+
}
|
|
286
|
+
};
|
|
287
|
+
__decorate([
|
|
288
|
+
(0, decorators_1.Transform)({ step: 1, name: 'retry', retry: { attempts: 3, delay: 5, backoff: 'fixed' } }),
|
|
289
|
+
__metadata("design:type", Function),
|
|
290
|
+
__metadata("design:paramtypes", []),
|
|
291
|
+
__metadata("design:returntype", void 0)
|
|
292
|
+
], RetryFixedPipeline.prototype, "retryStep", null);
|
|
293
|
+
RetryFixedPipeline = __decorate([
|
|
294
|
+
(0, decorators_1.Pipeline)('retry-fixed')
|
|
295
|
+
], RetryFixedPipeline);
|
|
296
|
+
describe('ETLService timeout and retry fixed', () => {
|
|
297
|
+
let etlService;
|
|
298
|
+
beforeEach(() => {
|
|
299
|
+
etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
|
|
300
|
+
});
|
|
301
|
+
it('times out when step exceeds timeoutMs', async () => {
|
|
302
|
+
const pipeline = new TimeoutPipeline();
|
|
303
|
+
await expect(etlService.execute(pipeline, {})).rejects.toThrow('timed out');
|
|
304
|
+
});
|
|
305
|
+
it('retries with fixed backoff', async () => {
|
|
306
|
+
const pipeline = new RetryFixedPipeline();
|
|
307
|
+
const result = await etlService.execute(pipeline, {});
|
|
308
|
+
expect(result.ok).toBe(true);
|
|
309
|
+
expect(pipeline.attempts).toBe(2);
|
|
310
|
+
});
|
|
311
|
+
it('emit catches handler errors', async () => {
|
|
312
|
+
const pipeline = new TestPipeline();
|
|
313
|
+
etlService.onStepComplete(() => {
|
|
314
|
+
throw new Error('Handler error');
|
|
315
|
+
});
|
|
316
|
+
const result = await etlService.execute(pipeline, { x: 5 });
|
|
317
|
+
expect(result).toEqual({ x: 10, y: 10 });
|
|
318
|
+
});
|
|
319
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { ETLService } from './etl.service';
|
|
2
|
+
/**
|
|
3
|
+
* Base class for pipelines - provides execute() method
|
|
4
|
+
* Extend this when using @Pipeline decorator for convenient execution
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* @Pipeline('user-enrichment')
|
|
9
|
+
* @Injectable()
|
|
10
|
+
* export class UserEnrichmentPipeline extends PipelineBase {
|
|
11
|
+
* constructor(etlService: ETLService) {
|
|
12
|
+
* super(etlService);
|
|
13
|
+
* }
|
|
14
|
+
* @Transform({ step: 1, name: 'normalize' }) async normalize(data) { ... }
|
|
15
|
+
* }
|
|
16
|
+
* // Usage: await pipeline.execute(rawUserData);
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export declare abstract class PipelineBase {
|
|
20
|
+
protected readonly etlService: ETLService;
|
|
21
|
+
constructor(etlService: ETLService);
|
|
22
|
+
execute<T = unknown>(input: unknown): Promise<T>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=pipeline.base.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.base.d.ts","sourceRoot":"","sources":["../../src/pipelines/pipeline.base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;GAgBG;AACH,8BAAsB,YAAY;IACpB,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU;gBAAtB,UAAU,EAAE,UAAU;IAE/C,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;CAGvD"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PipelineBase = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Base class for pipelines - provides execute() method
|
|
6
|
+
* Extend this when using @Pipeline decorator for convenient execution
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* @Pipeline('user-enrichment')
|
|
11
|
+
* @Injectable()
|
|
12
|
+
* export class UserEnrichmentPipeline extends PipelineBase {
|
|
13
|
+
* constructor(etlService: ETLService) {
|
|
14
|
+
* super(etlService);
|
|
15
|
+
* }
|
|
16
|
+
* @Transform({ step: 1, name: 'normalize' }) async normalize(data) { ... }
|
|
17
|
+
* }
|
|
18
|
+
* // Usage: await pipeline.execute(rawUserData);
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
class PipelineBase {
|
|
22
|
+
constructor(etlService) {
|
|
23
|
+
this.etlService = etlService;
|
|
24
|
+
}
|
|
25
|
+
async execute(input) {
|
|
26
|
+
return this.etlService.execute(this, input);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.PipelineBase = PipelineBase;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.base.test.d.ts","sourceRoot":"","sources":["../../src/pipelines/pipeline.base.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
const pipeline_base_1 = require("./pipeline.base");
|
|
13
|
+
const etl_service_1 = require("./etl.service");
|
|
14
|
+
const schema_validator_1 = require("../validators/schema.validator");
|
|
15
|
+
const decorators_1 = require("../decorators");
|
|
16
|
+
let TestPipeline = class TestPipeline extends pipeline_base_1.PipelineBase {
|
|
17
|
+
double(data) {
|
|
18
|
+
const x = data.x;
|
|
19
|
+
return { x: x * 2 };
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
__decorate([
|
|
23
|
+
(0, decorators_1.Transform)({ step: 1, name: 'double' }),
|
|
24
|
+
__metadata("design:type", Function),
|
|
25
|
+
__metadata("design:paramtypes", [Object]),
|
|
26
|
+
__metadata("design:returntype", void 0)
|
|
27
|
+
], TestPipeline.prototype, "double", null);
|
|
28
|
+
TestPipeline = __decorate([
|
|
29
|
+
(0, decorators_1.Pipeline)('base-test')
|
|
30
|
+
], TestPipeline);
|
|
31
|
+
describe('PipelineBase', () => {
|
|
32
|
+
it('execute runs pipeline via ETLService', async () => {
|
|
33
|
+
const etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
|
|
34
|
+
const pipeline = new TestPipeline(etlService);
|
|
35
|
+
const result = await pipeline.execute({ x: 5 });
|
|
36
|
+
expect(result).toEqual({ x: 10 });
|
|
37
|
+
});
|
|
38
|
+
});
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import type { RetryConfig, DLQConfig } from '../data.types';
|
|
2
|
+
type TransformFn = (data: unknown) => unknown | Promise<unknown>;
|
|
3
|
+
type ValidateFn = (data: unknown) => unknown;
|
|
4
|
+
type CatchFn = (data: unknown, error: Error) => unknown | Promise<unknown>;
|
|
5
|
+
type ConditionFn = (data: unknown) => boolean;
|
|
6
|
+
export interface PipelineStepConfig {
|
|
7
|
+
name: string;
|
|
8
|
+
transform?: TransformFn;
|
|
9
|
+
validate?: ValidateFn;
|
|
10
|
+
catch?: CatchFn;
|
|
11
|
+
when?: ConditionFn;
|
|
12
|
+
retry?: RetryConfig;
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
dlq?: DLQConfig;
|
|
15
|
+
/** Parallel transforms — all run concurrently, results merged */
|
|
16
|
+
parallel?: TransformFn[];
|
|
17
|
+
/** Branch on a condition: truthy → left builder, falsy → right builder */
|
|
18
|
+
branch?: {
|
|
19
|
+
condition: ConditionFn;
|
|
20
|
+
left: PipelineBuilder;
|
|
21
|
+
right: PipelineBuilder;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export interface SerializedStep {
|
|
25
|
+
name: string;
|
|
26
|
+
conditional?: boolean;
|
|
27
|
+
parallel?: boolean;
|
|
28
|
+
parallelCount?: number;
|
|
29
|
+
branch?: boolean;
|
|
30
|
+
retry?: RetryConfig;
|
|
31
|
+
timeoutMs?: number;
|
|
32
|
+
}
|
|
33
|
+
export interface PipelineDefinition {
|
|
34
|
+
name: string;
|
|
35
|
+
steps: SerializedStep[];
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* PipelineBuilder — immutable, fluent DSL for building ETL pipelines programmatically.
|
|
39
|
+
*
|
|
40
|
+
* Each builder method returns a **new** instance — the original is never mutated.
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* const pipeline = new PipelineBuilder()
|
|
44
|
+
* .setName('orders')
|
|
45
|
+
* .addTransform('normalize', (d) => ({ ...d, email: d.email.toLowerCase() }))
|
|
46
|
+
* .when((d) => d.active, (b) => b.addTransform('enrich', enrich))
|
|
47
|
+
* .addValidate('validate', validateFn);
|
|
48
|
+
*
|
|
49
|
+
* const result = await pipeline.execute(rawData);
|
|
50
|
+
*/
|
|
51
|
+
export declare class PipelineBuilder {
|
|
52
|
+
private readonly _name;
|
|
53
|
+
private readonly _steps;
|
|
54
|
+
constructor(name?: string, steps?: PipelineStepConfig[]);
|
|
55
|
+
setName(name: string): PipelineBuilder;
|
|
56
|
+
addTransform(name: string, transform: TransformFn, options?: {
|
|
57
|
+
when?: ConditionFn;
|
|
58
|
+
retry?: RetryConfig;
|
|
59
|
+
timeoutMs?: number;
|
|
60
|
+
dlq?: DLQConfig;
|
|
61
|
+
}): PipelineBuilder;
|
|
62
|
+
addValidate(name: string, validate: ValidateFn, options?: {
|
|
63
|
+
when?: ConditionFn;
|
|
64
|
+
}): PipelineBuilder;
|
|
65
|
+
/**
|
|
66
|
+
* Run multiple transforms concurrently. Results are merged (Object.assign) into
|
|
67
|
+
* the current data if they are objects, otherwise replaced with an array of results.
|
|
68
|
+
*/
|
|
69
|
+
parallel(name: string, transforms: TransformFn[]): PipelineBuilder;
|
|
70
|
+
/**
|
|
71
|
+
* Conditional branch: if `condition(data)` is true, run `thenBuilder` steps,
|
|
72
|
+
* otherwise run `elseBuilder` steps (default: identity).
|
|
73
|
+
*/
|
|
74
|
+
branch(name: string, condition: ConditionFn, thenBuilder: (b: PipelineBuilder) => PipelineBuilder, elseBuilder?: (b: PipelineBuilder) => PipelineBuilder): PipelineBuilder;
|
|
75
|
+
/**
|
|
76
|
+
* Attach a per-step error handler. If the previous step throws, `handler` is
|
|
77
|
+
* called with `(data, error)` and its return value becomes the new data.
|
|
78
|
+
*/
|
|
79
|
+
catch(handler: CatchFn): PipelineBuilder;
|
|
80
|
+
execute<T = unknown>(input: unknown): Promise<T>;
|
|
81
|
+
/** Serialize the pipeline definition to a plain object (steps with functions are omitted). */
|
|
82
|
+
toSchema(): PipelineDefinition;
|
|
83
|
+
build(): PipelineDefinition;
|
|
84
|
+
/** Create a fresh pipeline from a definition (transforms must be re-registered). */
|
|
85
|
+
static create(name?: string): PipelineBuilder;
|
|
86
|
+
/**
|
|
87
|
+
* @deprecated Use `new PipelineBuilder()` directly. Kept for backward compat.
|
|
88
|
+
* Note: this instance is now immutable — reset() returns a new empty builder.
|
|
89
|
+
*/
|
|
90
|
+
reset(): PipelineBuilder;
|
|
91
|
+
get name(): string;
|
|
92
|
+
get steps(): ReadonlyArray<PipelineStepConfig>;
|
|
93
|
+
}
|
|
94
|
+
export {};
|
|
95
|
+
//# sourceMappingURL=pipeline.builder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.builder.d.ts","sourceRoot":"","sources":["../../src/pipelines/pipeline.builder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE5D,KAAK,WAAW,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AACjE,KAAK,UAAU,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;AAC7C,KAAK,OAAO,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3E,KAAK,WAAW,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;AAE9C,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,WAAW,CAAC;IACxB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,0EAA0E;IAC1E,MAAM,CAAC,EAAE;QACP,SAAS,EAAE,WAAW,CAAC;QACvB,IAAI,EAAE,eAAe,CAAC;QACtB,KAAK,EAAE,eAAe,CAAC;KACxB,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AA8CD;;;;;;;;;;;;;GAaG;AACH,qBACa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;gBAE/C,IAAI,SAAqB,EAAE,KAAK,GAAE,kBAAkB,EAAO;IAOvE,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAMtC,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,WAAW,EACtB,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAC;QAAC,KAAK,CAAC,EAAE,WAAW,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,SAAS,CAAA;KAAO,GAC7F,eAAe;IAIlB,WAAW,CACT,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,UAAU,EACpB,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAA;KAAO,GACnC,eAAe;IAIlB;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,eAAe;IAIlE;;;OAGG;IACH,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,WAAW,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,eAAe,EACpD,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,eAAe,GACpD,eAAe;IASlB;;;OAGG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe;IAUlC,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IA4EtD,8FAA8F;IAC9F,QAAQ,IAAI,kBAAkB;IAgB9B,KAAK,IAAI,kBAAkB;IAI3B,oFAAoF;IACpF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe;IAI7C;;;OAGG;IACH,KAAK,IAAI,eAAe;IAIxB,IAAI,IAAI,IAAI,MAAM,CAEjB;IACD,IAAI,KAAK,IAAI,aAAa,CAAC,kBAAkB,CAAC,CAE7C;CACF"}
|