@hazeljs/data 0.2.0-beta.67 → 0.2.0-beta.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +175 -61
  2. package/dist/connectors/connector.interface.d.ts +29 -0
  3. package/dist/connectors/connector.interface.d.ts.map +1 -0
  4. package/dist/connectors/connector.interface.js +6 -0
  5. package/dist/connectors/csv.connector.d.ts +63 -0
  6. package/dist/connectors/csv.connector.d.ts.map +1 -0
  7. package/dist/connectors/csv.connector.js +147 -0
  8. package/dist/connectors/http.connector.d.ts +68 -0
  9. package/dist/connectors/http.connector.d.ts.map +1 -0
  10. package/dist/connectors/http.connector.js +131 -0
  11. package/dist/connectors/index.d.ts +7 -0
  12. package/dist/connectors/index.d.ts.map +1 -0
  13. package/dist/connectors/index.js +12 -0
  14. package/dist/connectors/memory.connector.d.ts +38 -0
  15. package/dist/connectors/memory.connector.d.ts.map +1 -0
  16. package/dist/connectors/memory.connector.js +56 -0
  17. package/dist/connectors/memory.connector.test.d.ts +2 -0
  18. package/dist/connectors/memory.connector.test.d.ts.map +1 -0
  19. package/dist/connectors/memory.connector.test.js +43 -0
  20. package/dist/data.types.d.ts +16 -0
  21. package/dist/data.types.d.ts.map +1 -1
  22. package/dist/decorators/index.d.ts +1 -0
  23. package/dist/decorators/index.d.ts.map +1 -1
  24. package/dist/decorators/index.js +8 -1
  25. package/dist/decorators/pii.decorator.d.ts +59 -0
  26. package/dist/decorators/pii.decorator.d.ts.map +1 -0
  27. package/dist/decorators/pii.decorator.js +197 -0
  28. package/dist/decorators/pii.decorator.test.d.ts +2 -0
  29. package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
  30. package/dist/decorators/pii.decorator.test.js +150 -0
  31. package/dist/decorators/pipeline.decorator.js +1 -1
  32. package/dist/decorators/pipeline.decorator.test.js +8 -0
  33. package/dist/decorators/transform.decorator.d.ts +9 -1
  34. package/dist/decorators/transform.decorator.d.ts.map +1 -1
  35. package/dist/decorators/transform.decorator.js +4 -0
  36. package/dist/decorators/validate.decorator.d.ts +5 -1
  37. package/dist/decorators/validate.decorator.d.ts.map +1 -1
  38. package/dist/decorators/validate.decorator.js +4 -0
  39. package/dist/flink.service.d.ts +30 -0
  40. package/dist/flink.service.d.ts.map +1 -1
  41. package/dist/flink.service.js +50 -2
  42. package/dist/index.d.ts +13 -7
  43. package/dist/index.d.ts.map +1 -1
  44. package/dist/index.js +36 -8
  45. package/dist/pipelines/etl.service.d.ts +41 -2
  46. package/dist/pipelines/etl.service.d.ts.map +1 -1
  47. package/dist/pipelines/etl.service.js +143 -6
  48. package/dist/pipelines/etl.service.test.js +215 -0
  49. package/dist/pipelines/pipeline.builder.d.ts +86 -13
  50. package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
  51. package/dist/pipelines/pipeline.builder.js +177 -27
  52. package/dist/pipelines/pipeline.builder.test.js +160 -12
  53. package/dist/pipelines/stream.service.test.js +49 -0
  54. package/dist/quality/quality.service.d.ts +67 -5
  55. package/dist/quality/quality.service.d.ts.map +1 -1
  56. package/dist/quality/quality.service.js +259 -20
  57. package/dist/quality/quality.service.test.js +94 -0
  58. package/dist/schema/schema.d.ts +92 -12
  59. package/dist/schema/schema.d.ts.map +1 -1
  60. package/dist/schema/schema.js +395 -83
  61. package/dist/schema/schema.test.js +292 -0
  62. package/dist/streaming/flink/flink.client.d.ts +41 -3
  63. package/dist/streaming/flink/flink.client.d.ts.map +1 -1
  64. package/dist/streaming/flink/flink.client.js +171 -8
  65. package/dist/streaming/flink/flink.client.test.js +2 -2
  66. package/dist/streaming/flink/flink.job.d.ts +2 -1
  67. package/dist/streaming/flink/flink.job.d.ts.map +1 -1
  68. package/dist/streaming/flink/flink.job.js +2 -2
  69. package/dist/streaming/stream.processor.d.ts +56 -2
  70. package/dist/streaming/stream.processor.d.ts.map +1 -1
  71. package/dist/streaming/stream.processor.js +149 -2
  72. package/dist/streaming/stream.processor.test.js +99 -0
  73. package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
  74. package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
  75. package/dist/streaming/stream.processor.windowing.test.js +69 -0
  76. package/dist/telemetry/telemetry.d.ts +124 -0
  77. package/dist/telemetry/telemetry.d.ts.map +1 -0
  78. package/dist/telemetry/telemetry.js +259 -0
  79. package/dist/telemetry/telemetry.test.d.ts +2 -0
  80. package/dist/telemetry/telemetry.test.d.ts.map +1 -0
  81. package/dist/telemetry/telemetry.test.js +51 -0
  82. package/dist/testing/index.d.ts +12 -0
  83. package/dist/testing/index.d.ts.map +1 -0
  84. package/dist/testing/index.js +18 -0
  85. package/dist/testing/pipeline-test-harness.d.ts +40 -0
  86. package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
  87. package/dist/testing/pipeline-test-harness.js +55 -0
  88. package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
  89. package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
  90. package/dist/testing/pipeline-test-harness.test.js +102 -0
  91. package/dist/testing/schema-faker.d.ts +32 -0
  92. package/dist/testing/schema-faker.d.ts.map +1 -0
  93. package/dist/testing/schema-faker.js +91 -0
  94. package/dist/testing/schema-faker.test.d.ts +2 -0
  95. package/dist/testing/schema-faker.test.d.ts.map +1 -0
  96. package/dist/testing/schema-faker.test.js +66 -0
  97. package/dist/transformers/built-in.transformers.test.js +28 -0
  98. package/dist/transformers/transformer.service.test.js +10 -0
  99. package/package.json +2 -2
@@ -17,13 +17,67 @@ const core_1 = require("@hazeljs/core");
17
17
  const decorators_1 = require("../decorators");
18
18
  const schema_validator_1 = require("../validators/schema.validator");
19
19
  const core_2 = __importDefault(require("@hazeljs/core"));
20
+ /** Runs a promise with a per-call timeout. */
21
+ async function withTimeout(promise, timeoutMs, stepName) {
22
+ return new Promise((resolve, reject) => {
23
+ const id = setTimeout(() => reject(new Error(`Step "${stepName}" timed out after ${timeoutMs}ms`)), timeoutMs);
24
+ promise.then((v) => {
25
+ clearTimeout(id);
26
+ resolve(v);
27
+ }, (e) => {
28
+ clearTimeout(id);
29
+ reject(e);
30
+ });
31
+ });
32
+ }
33
+ /** Executes fn with retry + exponential/fixed backoff. */
34
+ async function withRetry(fn, retry, stepName) {
35
+ const { attempts, delay = 500, backoff = 'fixed' } = retry;
36
+ let lastError = new Error('Unknown error');
37
+ for (let attempt = 1; attempt <= attempts; attempt++) {
38
+ try {
39
+ return await fn();
40
+ }
41
+ catch (err) {
42
+ lastError = err instanceof Error ? err : new Error(String(err));
43
+ if (attempt < attempts) {
44
+ const wait = backoff === 'exponential' ? delay * Math.pow(2, attempt - 1) : delay;
45
+ core_2.default.debug(`Step "${stepName}" attempt ${attempt} failed. Retrying in ${wait}ms...`);
46
+ await new Promise((r) => setTimeout(r, wait));
47
+ }
48
+ }
49
+ }
50
+ throw lastError;
51
+ }
20
52
  /**
21
- * ETL Service - Orchestrates pipeline execution
22
- * Executes steps sequentially: output from step N → input to step N+1
53
+ * ETL Service orchestrates pipeline execution.
54
+ *
55
+ * Features:
56
+ * - Sequential step execution (output N → input N+1)
57
+ * - Conditional steps via `when` predicate
58
+ * - Per-step retry with fixed/exponential backoff
59
+ * - Per-step execution timeout
60
+ * - Dead letter queue (DLQ) for graceful failure handling
61
+ * - Pipeline event hooks for observability
23
62
  */
24
63
  let ETLService = class ETLService {
25
64
  constructor(schemaValidator) {
26
65
  this.schemaValidator = schemaValidator;
66
+ this.eventHandlers = [];
67
+ }
68
+ /** Register a handler called after each step completes or fails. */
69
+ onStepComplete(handler) {
70
+ this.eventHandlers.push(handler);
71
+ }
72
+ emit(event) {
73
+ for (const h of this.eventHandlers) {
74
+ try {
75
+ h(event);
76
+ }
77
+ catch {
78
+ /* noop */
79
+ }
80
+ }
27
81
  }
28
82
  extractSteps(instance) {
29
83
  const steps = [];
@@ -41,6 +95,10 @@ let ETLService = class ETLService {
41
95
  name: transformMeta.name,
42
96
  type: 'transform',
43
97
  method: key,
98
+ when: transformMeta.when,
99
+ retry: transformMeta.retry,
100
+ timeoutMs: transformMeta.timeoutMs,
101
+ dlq: transformMeta.dlq,
44
102
  });
45
103
  }
46
104
  else if (validateMeta) {
@@ -50,6 +108,10 @@ let ETLService = class ETLService {
50
108
  type: 'validate',
51
109
  method: key,
52
110
  schema: validateMeta.schema,
111
+ when: validateMeta.when,
112
+ retry: validateMeta.retry,
113
+ timeoutMs: validateMeta.timeoutMs,
114
+ dlq: validateMeta.dlq,
53
115
  });
54
116
  }
55
117
  }
@@ -70,14 +132,89 @@ let ETLService = class ETLService {
70
132
  if (typeof fn !== 'function') {
71
133
  throw new Error(`Step ${step.name} method ${step.method} not found`);
72
134
  }
73
- if (step.type === 'validate' && step.schema) {
74
- data = this.schemaValidator.validate(step.schema, data);
135
+ // Conditional skip
136
+ if (step.when && !step.when(data)) {
137
+ core_2.default.debug(`Step "${step.name}" skipped (when predicate returned false)`);
138
+ this.emit({
139
+ pipeline: metadata?.name ?? 'unknown',
140
+ step: step.step,
141
+ stepName: step.name,
142
+ durationMs: 0,
143
+ success: true,
144
+ skipped: true,
145
+ });
146
+ continue;
147
+ }
148
+ const t0 = Date.now();
149
+ const runStep = async () => {
150
+ if (step.type === 'validate' && step.schema) {
151
+ data = this.schemaValidator.validate(step.schema, data);
152
+ }
153
+ const result = fn.call(pipelineInstance, data);
154
+ return result instanceof Promise ? await result : result;
155
+ };
156
+ try {
157
+ let stepPromise = step.retry ? withRetry(runStep, step.retry, step.name) : runStep();
158
+ if (step.timeoutMs) {
159
+ stepPromise = withTimeout(stepPromise, step.timeoutMs, step.name);
160
+ }
161
+ data = await stepPromise;
162
+ this.emit({
163
+ pipeline: metadata?.name ?? 'unknown',
164
+ step: step.step,
165
+ stepName: step.name,
166
+ durationMs: Date.now() - t0,
167
+ success: true,
168
+ });
169
+ }
170
+ catch (err) {
171
+ const error = err instanceof Error ? err : new Error(String(err));
172
+ this.emit({
173
+ pipeline: metadata?.name ?? 'unknown',
174
+ step: step.step,
175
+ stepName: step.name,
176
+ durationMs: Date.now() - t0,
177
+ success: false,
178
+ error: error.message,
179
+ });
180
+ if (step.dlq) {
181
+ core_2.default.debug(`Step "${step.name}" failed — routing to DLQ`);
182
+ await Promise.resolve(step.dlq.handler(data, error, step.name));
183
+ // Continue with unchanged data when routed to DLQ
184
+ }
185
+ else {
186
+ throw error;
187
+ }
75
188
  }
76
- const result = fn.call(pipelineInstance, data);
77
- data = result instanceof Promise ? await result : result;
78
189
  }
79
190
  return data;
80
191
  }
192
+ /**
193
+ * Execute multiple items through the pipeline in parallel.
194
+ * Items that fail are routed to the DLQ if configured, otherwise they propagate.
195
+ */
196
+ async executeBatch(pipelineInstance, items, options = {}) {
197
+ const { concurrency = 10 } = options;
198
+ const results = [];
199
+ const errors = [];
200
+ for (let i = 0; i < items.length; i += concurrency) {
201
+ const batch = items.slice(i, i + concurrency);
202
+ const settled = await Promise.allSettled(batch.map((item) => this.execute(pipelineInstance, item)));
203
+ for (let j = 0; j < settled.length; j++) {
204
+ const s = settled[j];
205
+ if (s.status === 'fulfilled') {
206
+ results.push(s.value);
207
+ }
208
+ else {
209
+ errors.push({
210
+ item: batch[j],
211
+ error: s.reason instanceof Error ? s.reason : new Error(String(s.reason)),
212
+ });
213
+ }
214
+ }
215
+ }
216
+ return { results, errors };
217
+ }
81
218
  };
82
219
  exports.ETLService = ETLService;
83
220
  exports.ETLService = ETLService = __decorate([
@@ -91,6 +91,11 @@ describe('ETLService', () => {
91
91
  const pipeline = new NoStepsPipeline();
92
92
  await expect(etlService.execute(pipeline, {})).rejects.toThrow('has no steps');
93
93
  });
94
+ it('throws when step method not found on instance', async () => {
95
+ const pipeline = new TestPipeline();
96
+ pipeline.double = null;
97
+ await expect(etlService.execute(pipeline, { x: 5 })).rejects.toThrow('not found');
98
+ });
94
99
  it('validates data at validate step', async () => {
95
100
  const pipeline = new TestPipeline();
96
101
  await expect(etlService.execute(pipeline, { x: -1 })).rejects.toThrow();
@@ -101,4 +106,214 @@ describe('ETLService', () => {
101
106
  expect(steps).toHaveLength(1);
102
107
  expect(steps[0].type).toBe('transform');
103
108
  });
109
+ it('onStepComplete emits events for each step', async () => {
110
+ const events = [];
111
+ etlService.onStepComplete((e) => events.push({ step: e.step, stepName: e.stepName, success: e.success }));
112
+ const pipeline = new TestPipeline();
113
+ await etlService.execute(pipeline, { x: 5 });
114
+ expect(events).toHaveLength(3);
115
+ expect(events.every((e) => e.success)).toBe(true);
116
+ expect(events.map((e) => e.stepName)).toEqual(['double', 'validate', 'add']);
117
+ });
118
+ it('executeBatch returns results and errors', async () => {
119
+ const pipeline = new TestPipeline();
120
+ const { results, errors } = await etlService.executeBatch(pipeline, [{ x: 1 }, { x: -1 }, { x: 2 }], { concurrency: 2 });
121
+ expect(results).toHaveLength(2);
122
+ expect(results[0]).toEqual({ x: 2, y: 10 });
123
+ expect(results[1]).toEqual({ x: 4, y: 10 });
124
+ expect(errors).toHaveLength(1);
125
+ expect(errors[0].item).toEqual({ x: -1 });
126
+ });
127
+ it('executeBatch wraps non-Error rejections', async () => {
128
+ let ThrowStringPipeline = class ThrowStringPipeline {
129
+ fail() {
130
+ throw 'string rejection';
131
+ }
132
+ };
133
+ __decorate([
134
+ (0, decorators_1.Transform)({ step: 1, name: 'fail' }),
135
+ __metadata("design:type", Function),
136
+ __metadata("design:paramtypes", []),
137
+ __metadata("design:returntype", void 0)
138
+ ], ThrowStringPipeline.prototype, "fail", null);
139
+ ThrowStringPipeline = __decorate([
140
+ (0, decorators_1.Pipeline)('throw-string')
141
+ ], ThrowStringPipeline);
142
+ const { errors } = await etlService.executeBatch(new ThrowStringPipeline(), [{}]);
143
+ expect(errors).toHaveLength(1);
144
+ expect(errors[0].error).toBeInstanceOf(Error);
145
+ expect(errors[0].error.message).toContain('string rejection');
146
+ });
147
+ });
148
+ let ConditionalPipeline = class ConditionalPipeline {
149
+ always(data) {
150
+ return { ...data, step: 1 };
151
+ }
152
+ whenStep(data) {
153
+ return { ...data, step: 2 };
154
+ }
155
+ };
156
+ __decorate([
157
+ (0, decorators_1.Transform)({ step: 1, name: 'always' }),
158
+ __metadata("design:type", Function),
159
+ __metadata("design:paramtypes", [Object]),
160
+ __metadata("design:returntype", void 0)
161
+ ], ConditionalPipeline.prototype, "always", null);
162
+ __decorate([
163
+ (0, decorators_1.Transform)({
164
+ step: 2,
165
+ name: 'when-true',
166
+ when: (d) => d.skip !== true,
167
+ }),
168
+ __metadata("design:type", Function),
169
+ __metadata("design:paramtypes", [Object]),
170
+ __metadata("design:returntype", void 0)
171
+ ], ConditionalPipeline.prototype, "whenStep", null);
172
+ ConditionalPipeline = __decorate([
173
+ (0, decorators_1.Pipeline)('conditional-pipeline')
174
+ ], ConditionalPipeline);
175
+ describe('ETLService conditional steps', () => {
176
+ let etlService;
177
+ beforeEach(() => {
178
+ etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
179
+ });
180
+ it('skips step when when predicate returns false', async () => {
181
+ const pipeline = new ConditionalPipeline();
182
+ const events = [];
183
+ etlService.onStepComplete((e) => events.push({ stepName: e.stepName, skipped: e.skipped }));
184
+ const result = await etlService.execute(pipeline, {
185
+ skip: true,
186
+ });
187
+ expect(result.step).toBe(1);
188
+ expect(events.find((e) => e.stepName === 'when-true')?.skipped).toBe(true);
189
+ });
190
+ it('runs step when when predicate returns true', async () => {
191
+ const pipeline = new ConditionalPipeline();
192
+ const result = await etlService.execute(pipeline, {});
193
+ expect(result.step).toBe(2);
194
+ });
195
+ });
196
+ let RetryPipeline = class RetryPipeline {
197
+ constructor() {
198
+ this.attempts = 0;
199
+ }
200
+ retryStep(data) {
201
+ this.attempts++;
202
+ if (this.attempts < 2)
203
+ throw new Error('Fail');
204
+ return { ...data, ok: true };
205
+ }
206
+ };
207
+ __decorate([
208
+ (0, decorators_1.Transform)({ step: 1, name: 'retry', retry: { attempts: 3, delay: 10, backoff: 'exponential' } }),
209
+ __metadata("design:type", Function),
210
+ __metadata("design:paramtypes", [Object]),
211
+ __metadata("design:returntype", void 0)
212
+ ], RetryPipeline.prototype, "retryStep", null);
213
+ RetryPipeline = __decorate([
214
+ (0, decorators_1.Pipeline)('retry-pipeline')
215
+ ], RetryPipeline);
216
+ const dlqCaptured = [];
217
+ let DLQPipeline = class DLQPipeline {
218
+ failStep(data) {
219
+ if (data.skip)
220
+ return data;
221
+ throw new Error('DLQ test error');
222
+ }
223
+ };
224
+ __decorate([
225
+ (0, decorators_1.Transform)({
226
+ step: 1,
227
+ name: 'fail',
228
+ dlq: {
229
+ handler: (item, err) => {
230
+ dlqCaptured.push({ item, error: err.message });
231
+ },
232
+ },
233
+ }),
234
+ __metadata("design:type", Function),
235
+ __metadata("design:paramtypes", [Object]),
236
+ __metadata("design:returntype", void 0)
237
+ ], DLQPipeline.prototype, "failStep", null);
238
+ DLQPipeline = __decorate([
239
+ (0, decorators_1.Pipeline)('dlq-pipeline')
240
+ ], DLQPipeline);
241
+ describe('ETLService retry and DLQ', () => {
242
+ let etlService;
243
+ beforeEach(() => {
244
+ etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
245
+ dlqCaptured.length = 0;
246
+ });
247
+ it('retries step on failure', async () => {
248
+ const pipeline = new RetryPipeline();
249
+ const result = await etlService.execute(pipeline, {});
250
+ expect(result.ok).toBe(true);
251
+ expect(pipeline.attempts).toBe(2);
252
+ });
253
+ it('dlq receives failed item and pipeline continues with unchanged data', async () => {
254
+ const pipeline = new DLQPipeline();
255
+ const result = await etlService.execute(pipeline, { x: 1 });
256
+ expect(dlqCaptured).toHaveLength(1);
257
+ expect(dlqCaptured[0].error).toBe('DLQ test error');
258
+ expect(result).toEqual({ x: 1 });
259
+ });
260
+ });
261
+ let TimeoutPipeline = class TimeoutPipeline {
262
+ async slowStep() {
263
+ await new Promise((r) => setTimeout(r, 200));
264
+ return { done: true };
265
+ }
266
+ };
267
+ __decorate([
268
+ (0, decorators_1.Transform)({ step: 1, name: 'slow', timeoutMs: 50 }),
269
+ __metadata("design:type", Function),
270
+ __metadata("design:paramtypes", []),
271
+ __metadata("design:returntype", Promise)
272
+ ], TimeoutPipeline.prototype, "slowStep", null);
273
+ TimeoutPipeline = __decorate([
274
+ (0, decorators_1.Pipeline)('timeout-pipeline')
275
+ ], TimeoutPipeline);
276
+ let RetryFixedPipeline = class RetryFixedPipeline {
277
+ constructor() {
278
+ this.attempts = 0;
279
+ }
280
+ retryStep() {
281
+ this.attempts++;
282
+ if (this.attempts < 2)
283
+ throw new Error('Fail');
284
+ return { ok: true };
285
+ }
286
+ };
287
+ __decorate([
288
+ (0, decorators_1.Transform)({ step: 1, name: 'retry', retry: { attempts: 3, delay: 5, backoff: 'fixed' } }),
289
+ __metadata("design:type", Function),
290
+ __metadata("design:paramtypes", []),
291
+ __metadata("design:returntype", void 0)
292
+ ], RetryFixedPipeline.prototype, "retryStep", null);
293
+ RetryFixedPipeline = __decorate([
294
+ (0, decorators_1.Pipeline)('retry-fixed')
295
+ ], RetryFixedPipeline);
296
+ describe('ETLService timeout and retry fixed', () => {
297
+ let etlService;
298
+ beforeEach(() => {
299
+ etlService = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
300
+ });
301
+ it('times out when step exceeds timeoutMs', async () => {
302
+ const pipeline = new TimeoutPipeline();
303
+ await expect(etlService.execute(pipeline, {})).rejects.toThrow('timed out');
304
+ });
305
+ it('retries with fixed backoff', async () => {
306
+ const pipeline = new RetryFixedPipeline();
307
+ const result = await etlService.execute(pipeline, {});
308
+ expect(result.ok).toBe(true);
309
+ expect(pipeline.attempts).toBe(2);
310
+ });
311
+ it('emit catches handler errors', async () => {
312
+ const pipeline = new TestPipeline();
313
+ etlService.onStepComplete(() => {
314
+ throw new Error('Handler error');
315
+ });
316
+ const result = await etlService.execute(pipeline, { x: 5 });
317
+ expect(result).toEqual({ x: 10, y: 10 });
318
+ });
104
319
  });
@@ -1,22 +1,95 @@
1
+ import type { RetryConfig, DLQConfig } from '../data.types';
2
+ type TransformFn = (data: unknown) => unknown | Promise<unknown>;
3
+ type ValidateFn = (data: unknown) => unknown;
4
+ type CatchFn = (data: unknown, error: Error) => unknown | Promise<unknown>;
5
+ type ConditionFn = (data: unknown) => boolean;
1
6
  export interface PipelineStepConfig {
2
7
  name: string;
3
- transform?: (data: unknown) => unknown | Promise<unknown>;
4
- validate?: (data: unknown) => unknown;
8
+ transform?: TransformFn;
9
+ validate?: ValidateFn;
10
+ catch?: CatchFn;
11
+ when?: ConditionFn;
12
+ retry?: RetryConfig;
13
+ timeoutMs?: number;
14
+ dlq?: DLQConfig;
15
+ /** Parallel transforms — all run concurrently, results merged */
16
+ parallel?: TransformFn[];
17
+ /** Branch on a condition: truthy → left builder, falsy → right builder */
18
+ branch?: {
19
+ condition: ConditionFn;
20
+ left: PipelineBuilder;
21
+ right: PipelineBuilder;
22
+ };
23
+ }
24
+ export interface SerializedStep {
25
+ name: string;
26
+ conditional?: boolean;
27
+ parallel?: boolean;
28
+ parallelCount?: number;
29
+ branch?: boolean;
30
+ retry?: RetryConfig;
31
+ timeoutMs?: number;
32
+ }
33
+ export interface PipelineDefinition {
34
+ name: string;
35
+ steps: SerializedStep[];
5
36
  }
6
37
  /**
7
- * Pipeline Builder - DSL for building pipelines programmatically
38
+ * PipelineBuilder immutable, fluent DSL for building ETL pipelines programmatically.
39
+ *
40
+ * Each builder method returns a **new** instance — the original is never mutated.
41
+ *
42
+ * @example
43
+ * const pipeline = new PipelineBuilder()
44
+ * .setName('orders')
45
+ * .addTransform('normalize', (d) => ({ ...d, email: d.email.toLowerCase() }))
46
+ * .when((d) => d.active, (b) => b.addTransform('enrich', enrich))
47
+ * .addValidate('validate', validateFn);
48
+ *
49
+ * const result = await pipeline.execute(rawData);
8
50
  */
9
51
  export declare class PipelineBuilder {
10
- private steps;
11
- private name;
12
- setName(name: string): this;
13
- addTransform(name: string, transform: (data: unknown) => unknown | Promise<unknown>): this;
14
- addValidate(name: string, validate: (data: unknown) => unknown): this;
52
+ private readonly _name;
53
+ private readonly _steps;
54
+ constructor(name?: string, steps?: PipelineStepConfig[]);
55
+ setName(name: string): PipelineBuilder;
56
+ addTransform(name: string, transform: TransformFn, options?: {
57
+ when?: ConditionFn;
58
+ retry?: RetryConfig;
59
+ timeoutMs?: number;
60
+ dlq?: DLQConfig;
61
+ }): PipelineBuilder;
62
+ addValidate(name: string, validate: ValidateFn, options?: {
63
+ when?: ConditionFn;
64
+ }): PipelineBuilder;
65
+ /**
66
+ * Run multiple transforms concurrently. Results are merged (Object.assign) into
67
+ * the current data if they are objects, otherwise replaced with an array of results.
68
+ */
69
+ parallel(name: string, transforms: TransformFn[]): PipelineBuilder;
70
+ /**
71
+ * Conditional branch: if `condition(data)` is true, run `thenBuilder` steps,
72
+ * otherwise run `elseBuilder` steps (default: identity).
73
+ */
74
+ branch(name: string, condition: ConditionFn, thenBuilder: (b: PipelineBuilder) => PipelineBuilder, elseBuilder?: (b: PipelineBuilder) => PipelineBuilder): PipelineBuilder;
75
+ /**
76
+ * Attach a per-step error handler. If the previous step throws, `handler` is
77
+ * called with `(data, error)` and its return value becomes the new data.
78
+ */
79
+ catch(handler: CatchFn): PipelineBuilder;
15
80
  execute<T = unknown>(input: unknown): Promise<T>;
16
- build(): {
17
- name: string;
18
- steps: PipelineStepConfig[];
19
- };
20
- reset(): this;
81
+ /** Serialize the pipeline definition to a plain object (steps with functions are omitted). */
82
+ toSchema(): PipelineDefinition;
83
+ build(): PipelineDefinition;
84
+ /** Create a fresh pipeline from a definition (transforms must be re-registered). */
85
+ static create(name?: string): PipelineBuilder;
86
+ /**
87
+ * @deprecated Use `new PipelineBuilder()` directly. Kept for backward compat.
88
+ * Note: this instance is now immutable — reset() returns a new empty builder.
89
+ */
90
+ reset(): PipelineBuilder;
91
+ get name(): string;
92
+ get steps(): ReadonlyArray<PipelineStepConfig>;
21
93
  }
94
+ export {};
22
95
  //# sourceMappingURL=pipeline.builder.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"pipeline.builder.d.ts","sourceRoot":"","sources":["../../src/pipelines/pipeline.builder.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;CACvC;AAED;;GAEG;AACH,qBACa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,IAAI,CAAsB;IAElC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAK3B,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI;IAK1F,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GAAG,IAAI;IAK/D,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAkBtD,KAAK,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,kBAAkB,EAAE,CAAA;KAAE;IAItD,KAAK,IAAI,IAAI;CAKd"}
1
+ {"version":3,"file":"pipeline.builder.d.ts","sourceRoot":"","sources":["../../src/pipelines/pipeline.builder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE5D,KAAK,WAAW,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AACjE,KAAK,UAAU,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;AAC7C,KAAK,OAAO,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3E,KAAK,WAAW,GAAG,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;AAE9C,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,WAAW,CAAC;IACxB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,0EAA0E;IAC1E,MAAM,CAAC,EAAE;QACP,SAAS,EAAE,WAAW,CAAC;QACvB,IAAI,EAAE,eAAe,CAAC;QACtB,KAAK,EAAE,eAAe,CAAC;KACxB,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AA8CD;;;;;;;;;;;;;GAaG;AACH,qBACa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;gBAE/C,IAAI,SAAqB,EAAE,KAAK,GAAE,kBAAkB,EAAO;IAOvE,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAMtC,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,WAAW,EACtB,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAC;QAAC,KAAK,CAAC,EAAE,WAAW,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,SAAS,CAAA;KAAO,GAC7F,eAAe;IAIlB,WAAW,CACT,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,UAAU,EACpB,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,WAAW,CAAA;KAAO,GACnC,eAAe;IAIlB;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,eAAe;IAIlE;;;OAGG;IACH,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,WAAW,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,eAAe,EACpD,WAAW,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,KAAK,eAAe,GACpD,eAAe;IASlB;;;OAGG;IACH,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe;IAUlC,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IA4EtD,8FAA8F;IAC9F,QAAQ,IAAI,kBAAkB;IAgB9B,KAAK,IAAI,kBAAkB;IAI3B,oFAAoF;IACpF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,eAAe;IAI7C;;;OAGG;IACH,KAAK,IAAI,eAAe;IAIxB,IAAI,IAAI,IAAI,MAAM,CAEjB;IACD,IAAI,KAAK,IAAI,aAAa,CAAC,kBAAkB,CAAC,CAE7C;CACF"}