@hazeljs/data 0.2.0-beta.67 → 0.2.0-beta.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +175 -61
  2. package/dist/connectors/connector.interface.d.ts +29 -0
  3. package/dist/connectors/connector.interface.d.ts.map +1 -0
  4. package/dist/connectors/connector.interface.js +6 -0
  5. package/dist/connectors/csv.connector.d.ts +63 -0
  6. package/dist/connectors/csv.connector.d.ts.map +1 -0
  7. package/dist/connectors/csv.connector.js +147 -0
  8. package/dist/connectors/http.connector.d.ts +68 -0
  9. package/dist/connectors/http.connector.d.ts.map +1 -0
  10. package/dist/connectors/http.connector.js +131 -0
  11. package/dist/connectors/index.d.ts +7 -0
  12. package/dist/connectors/index.d.ts.map +1 -0
  13. package/dist/connectors/index.js +12 -0
  14. package/dist/connectors/memory.connector.d.ts +38 -0
  15. package/dist/connectors/memory.connector.d.ts.map +1 -0
  16. package/dist/connectors/memory.connector.js +56 -0
  17. package/dist/connectors/memory.connector.test.d.ts +2 -0
  18. package/dist/connectors/memory.connector.test.d.ts.map +1 -0
  19. package/dist/connectors/memory.connector.test.js +43 -0
  20. package/dist/data.types.d.ts +16 -0
  21. package/dist/data.types.d.ts.map +1 -1
  22. package/dist/decorators/index.d.ts +1 -0
  23. package/dist/decorators/index.d.ts.map +1 -1
  24. package/dist/decorators/index.js +8 -1
  25. package/dist/decorators/pii.decorator.d.ts +59 -0
  26. package/dist/decorators/pii.decorator.d.ts.map +1 -0
  27. package/dist/decorators/pii.decorator.js +197 -0
  28. package/dist/decorators/pii.decorator.test.d.ts +2 -0
  29. package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
  30. package/dist/decorators/pii.decorator.test.js +150 -0
  31. package/dist/decorators/pipeline.decorator.js +1 -1
  32. package/dist/decorators/pipeline.decorator.test.js +8 -0
  33. package/dist/decorators/transform.decorator.d.ts +9 -1
  34. package/dist/decorators/transform.decorator.d.ts.map +1 -1
  35. package/dist/decorators/transform.decorator.js +4 -0
  36. package/dist/decorators/validate.decorator.d.ts +5 -1
  37. package/dist/decorators/validate.decorator.d.ts.map +1 -1
  38. package/dist/decorators/validate.decorator.js +4 -0
  39. package/dist/flink.service.d.ts +30 -0
  40. package/dist/flink.service.d.ts.map +1 -1
  41. package/dist/flink.service.js +50 -2
  42. package/dist/index.d.ts +13 -7
  43. package/dist/index.d.ts.map +1 -1
  44. package/dist/index.js +36 -8
  45. package/dist/pipelines/etl.service.d.ts +41 -2
  46. package/dist/pipelines/etl.service.d.ts.map +1 -1
  47. package/dist/pipelines/etl.service.js +143 -6
  48. package/dist/pipelines/etl.service.test.js +215 -0
  49. package/dist/pipelines/pipeline.builder.d.ts +86 -13
  50. package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
  51. package/dist/pipelines/pipeline.builder.js +177 -27
  52. package/dist/pipelines/pipeline.builder.test.js +160 -12
  53. package/dist/pipelines/stream.service.test.js +49 -0
  54. package/dist/quality/quality.service.d.ts +67 -5
  55. package/dist/quality/quality.service.d.ts.map +1 -1
  56. package/dist/quality/quality.service.js +259 -20
  57. package/dist/quality/quality.service.test.js +94 -0
  58. package/dist/schema/schema.d.ts +92 -12
  59. package/dist/schema/schema.d.ts.map +1 -1
  60. package/dist/schema/schema.js +395 -83
  61. package/dist/schema/schema.test.js +292 -0
  62. package/dist/streaming/flink/flink.client.d.ts +41 -3
  63. package/dist/streaming/flink/flink.client.d.ts.map +1 -1
  64. package/dist/streaming/flink/flink.client.js +171 -8
  65. package/dist/streaming/flink/flink.client.test.js +2 -2
  66. package/dist/streaming/flink/flink.job.d.ts +2 -1
  67. package/dist/streaming/flink/flink.job.d.ts.map +1 -1
  68. package/dist/streaming/flink/flink.job.js +2 -2
  69. package/dist/streaming/stream.processor.d.ts +56 -2
  70. package/dist/streaming/stream.processor.d.ts.map +1 -1
  71. package/dist/streaming/stream.processor.js +149 -2
  72. package/dist/streaming/stream.processor.test.js +99 -0
  73. package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
  74. package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
  75. package/dist/streaming/stream.processor.windowing.test.js +69 -0
  76. package/dist/telemetry/telemetry.d.ts +124 -0
  77. package/dist/telemetry/telemetry.d.ts.map +1 -0
  78. package/dist/telemetry/telemetry.js +259 -0
  79. package/dist/telemetry/telemetry.test.d.ts +2 -0
  80. package/dist/telemetry/telemetry.test.d.ts.map +1 -0
  81. package/dist/telemetry/telemetry.test.js +51 -0
  82. package/dist/testing/index.d.ts +12 -0
  83. package/dist/testing/index.d.ts.map +1 -0
  84. package/dist/testing/index.js +18 -0
  85. package/dist/testing/pipeline-test-harness.d.ts +40 -0
  86. package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
  87. package/dist/testing/pipeline-test-harness.js +55 -0
  88. package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
  89. package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
  90. package/dist/testing/pipeline-test-harness.test.js +102 -0
  91. package/dist/testing/schema-faker.d.ts +32 -0
  92. package/dist/testing/schema-faker.d.ts.map +1 -0
  93. package/dist/testing/schema-faker.js +91 -0
  94. package/dist/testing/schema-faker.test.d.ts +2 -0
  95. package/dist/testing/schema-faker.test.d.ts.map +1 -0
  96. package/dist/testing/schema-faker.test.js +66 -0
  97. package/dist/transformers/built-in.transformers.test.js +28 -0
  98. package/dist/transformers/transformer.service.test.js +10 -0
  99. package/package.json +2 -2
@@ -1 +1 @@
1
- {"version":3,"file":"stream.processor.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD;;;GAGG;AACH,qBAAa,eAAe;IACd,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,EAAE,UAAU;IAE7C,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAIlE,aAAa,CAAC,CAAC,EACpB,gBAAgB,EAAE,MAAM,EACxB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAC7B,cAAc,CAAC,CAAC,CAAC;CAUrB"}
1
+ {"version":3,"file":"stream.processor.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;GASG;AACH,qBAAa,eAAe;IACd,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,EAAE,UAAU;IAE7C,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAIlE,aAAa,CAAC,CAAC,EACpB,gBAAgB,EAAE,MAAM,EACxB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAC7B,cAAc,CAAC,CAAC,CAAC;IAapB;;;;;;OAMG;IACI,cAAc,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,EACtD,QAAQ,EAAE,MAAM,GACf,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IA2BnC;;;;;;OAMG;IACI,aAAa,CAAC,CAAC,EACpB,MAAM,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,EACtD,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IA6BnC;;;;;;;OAOG;IACI,aAAa,CAAC,CAAC,EACpB,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,EACxB,MAAM,EAAE,MAAM,EACd,YAAY,GAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAyB,GACnD,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAwBnC;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAC1B,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EACtB,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,EACvB,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC5B,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC7B,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,EAC1B,QAAQ,SAAS,GAChB,cAAc,CAAC,GAAG,CAAC;CAuCvB"}
@@ -6,8 +6,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.StreamProcessor = void 0;
7
7
  const core_1 = __importDefault(require("@hazeljs/core"));
8
8
  /**
9
- * Stream Processor - In-process stream processing logic
10
- * Processes items through pipeline without Flink (for testing/simple use cases)
9
+ * Stream Processor in-process stream processing.
10
+ *
11
+ * Features:
12
+ * - `processItem` / `processStream` — single-item and async-generator streaming
13
+ * - `tumblingWindow` — non-overlapping time windows
14
+ * - `slidingWindow` — overlapping time windows
15
+ * - `sessionWindow` — gap-based grouping (items grouped when gap > idleMs)
16
+ * - `joinStreams` — merge two async iterables by matching key
11
17
  */
12
18
  class StreamProcessor {
13
19
  constructor(etlService) {
@@ -27,5 +33,146 @@ class StreamProcessor {
27
33
  }
28
34
  }
29
35
  }
36
+ // ─── Windowing ─────────────────────────────────────────────────────────────
37
+ /**
38
+ * Tumbling window — collects items into non-overlapping fixed-size time windows.
39
+ * Each item is assigned to exactly one window.
40
+ *
41
+ * @param source Async iterable of `{ value: T; timestamp: number }` items
42
+ * @param windowMs Window duration in milliseconds
43
+ */
44
+ async *tumblingWindow(source, windowMs) {
45
+ let windowStart = null;
46
+ let buffer = [];
47
+ for await (const item of source) {
48
+ if (windowStart === null) {
49
+ windowStart = item.timestamp - (item.timestamp % windowMs);
50
+ }
51
+ const itemWindow = item.timestamp - (item.timestamp % windowMs);
52
+ if (itemWindow > windowStart) {
53
+ if (buffer.length > 0) {
54
+ yield { items: buffer, windowStart, windowEnd: windowStart + windowMs };
55
+ }
56
+ windowStart = itemWindow;
57
+ buffer = [];
58
+ }
59
+ buffer.push(item.value);
60
+ }
61
+ if (buffer.length > 0 && windowStart !== null) {
62
+ yield { items: buffer, windowStart, windowEnd: windowStart + windowMs };
63
+ }
64
+ }
65
+ /**
66
+ * Sliding window — each item may appear in multiple windows.
67
+ *
68
+ * @param source Async iterable of `{ value: T; timestamp: number }` items
69
+ * @param windowMs Window size in milliseconds
70
+ * @param slideMs Slide interval in milliseconds (how often a new window starts)
71
+ */
72
+ async *slidingWindow(source, windowMs, slideMs) {
73
+ const buffer = [];
74
+ const emittedWindows = new Set();
75
+ for await (const item of source) {
76
+ buffer.push(item);
77
+ // Remove items outside the largest possible window
78
+ const oldest = item.timestamp - windowMs;
79
+ while (buffer.length > 0 && buffer[0].timestamp < oldest) {
80
+ buffer.shift();
81
+ }
82
+ // Determine which sliding window this item triggers
83
+ const windowKey = Math.floor(item.timestamp / slideMs) * slideMs;
84
+ if (!emittedWindows.has(windowKey)) {
85
+ emittedWindows.add(windowKey);
86
+ const windowStart = windowKey;
87
+ const windowEnd = windowStart + windowMs;
88
+ const items = buffer
89
+ .filter((b) => b.timestamp >= windowStart && b.timestamp < windowEnd)
90
+ .map((b) => b.value);
91
+ if (items.length > 0) {
92
+ yield { items, windowStart, windowEnd };
93
+ }
94
+ }
95
+ }
96
+ }
97
+ /**
98
+ * Session window — groups items by inactivity gaps.
99
+ * A new window starts when no item is received for longer than `idleMs`.
100
+ *
101
+ * @param source Async iterable of items (with optional `.timestamp`)
102
+ * @param idleMs Gap duration that triggers a new session (default: 30_000ms)
103
+ * @param getTimestamp Function to extract timestamp from an item (default: Date.now())
104
+ */
105
+ async *sessionWindow(source, idleMs, getTimestamp = () => Date.now()) {
106
+ let buffer = [];
107
+ let lastTimestamp = null;
108
+ let windowStart = null;
109
+ for await (const item of source) {
110
+ const ts = getTimestamp(item);
111
+ if (lastTimestamp !== null && ts - lastTimestamp > idleMs && buffer.length > 0) {
112
+ yield { items: buffer, windowStart: windowStart, windowEnd: lastTimestamp };
113
+ buffer = [];
114
+ windowStart = null;
115
+ }
116
+ if (windowStart === null)
117
+ windowStart = ts;
118
+ buffer.push(item);
119
+ lastTimestamp = ts;
120
+ }
121
+ if (buffer.length > 0 && windowStart !== null && lastTimestamp !== null) {
122
+ yield { items: buffer, windowStart, windowEnd: lastTimestamp };
123
+ }
124
+ }
125
+ /**
126
+ * Join two async streams by a key function.
127
+ * Items from `left` are buffered; when a matching `right` item arrives, they are emitted together.
128
+ *
129
+ * @param left Left stream
130
+ * @param right Right stream
131
+ * @param leftKey Extract join key from left items
132
+ * @param rightKey Extract join key from right items
133
+ * @param merge Combine matched left + right items
134
+ * @param windowMs How long to buffer unmatched left items (default: 60_000ms)
135
+ */
136
+ async *joinStreams(left, right, leftKey, rightKey, merge, windowMs = 60000) {
137
+ const leftBuffer = new Map();
138
+ const rightBuffer = new Map();
139
+ const now = () => Date.now();
140
+ // Flatten both streams into a single tagged stream sequentially
141
+ // For a true concurrent join, use a more complex scheduler —
142
+ // this simpler version drains left first, then right, checking buffers on both sides.
143
+ for await (const lItem of left) {
144
+ const key = leftKey(lItem);
145
+ if (rightBuffer.has(key)) {
146
+ const { item: rItem } = rightBuffer.get(key);
147
+ rightBuffer.delete(key);
148
+ yield merge(lItem, rItem);
149
+ }
150
+ else {
151
+ leftBuffer.set(key, { item: lItem, ts: now() });
152
+ }
153
+ // Expire stale unmatched left items
154
+ const expiry = now() - windowMs;
155
+ for (const [k, v] of leftBuffer) {
156
+ if (v.ts < expiry)
157
+ leftBuffer.delete(k);
158
+ }
159
+ }
160
+ for await (const rItem of right) {
161
+ const key = rightKey(rItem);
162
+ if (leftBuffer.has(key)) {
163
+ const { item: lItem } = leftBuffer.get(key);
164
+ leftBuffer.delete(key);
165
+ yield merge(lItem, rItem);
166
+ }
167
+ else {
168
+ rightBuffer.set(key, { item: rItem, ts: now() });
169
+ }
170
+ const expiry = now() - windowMs;
171
+ for (const [k, v] of rightBuffer) {
172
+ if (v.ts < expiry)
173
+ rightBuffer.delete(k);
174
+ }
175
+ }
176
+ }
30
177
  }
31
178
  exports.StreamProcessor = StreamProcessor;
@@ -49,4 +49,103 @@ describe('StreamProcessor', () => {
49
49
  }
50
50
  expect(results).toEqual([{ v: 2 }, { v: 3 }]);
51
51
  });
52
+ it('processStream throws when item fails', async () => {
53
+ let FailPipeline = class FailPipeline {
54
+ fail() {
55
+ throw new Error('Item failed');
56
+ }
57
+ };
58
+ __decorate([
59
+ (0, decorators_1.Transform)({ step: 1, name: 'fail' }),
60
+ __metadata("design:type", Function),
61
+ __metadata("design:paramtypes", []),
62
+ __metadata("design:returntype", void 0)
63
+ ], FailPipeline.prototype, "fail", null);
64
+ FailPipeline = __decorate([
65
+ (0, decorators_1.Pipeline)('fail-proc')
66
+ ], FailPipeline);
67
+ async function* gen() {
68
+ yield {};
69
+ }
70
+ await expect((async () => {
71
+ for await (const _ of processor.processStream(new FailPipeline(), gen())) {
72
+ break;
73
+ }
74
+ })()).rejects.toThrow('Item failed');
75
+ });
76
+ it('tumblingWindow groups items by time window', async () => {
77
+ async function* source() {
78
+ yield { value: 1, timestamp: 100 };
79
+ yield { value: 2, timestamp: 150 };
80
+ yield { value: 3, timestamp: 250 };
81
+ }
82
+ const batches = [];
83
+ for await (const b of processor.tumblingWindow(source(), 100)) {
84
+ batches.push({ items: b.items, windowStart: b.windowStart });
85
+ }
86
+ expect(batches).toHaveLength(2);
87
+ expect(batches[0].items).toEqual([1, 2]);
88
+ expect(batches[0].windowStart).toBe(100);
89
+ expect(batches[1].items).toEqual([3]);
90
+ expect(batches[1].windowStart).toBe(200);
91
+ });
92
+ it('slidingWindow emits overlapping windows', async () => {
93
+ async function* source() {
94
+ yield { value: 1, timestamp: 0 };
95
+ yield { value: 2, timestamp: 50 };
96
+ }
97
+ const batches = [];
98
+ for await (const b of processor.slidingWindow(source(), 100, 50)) {
99
+ batches.push(b);
100
+ }
101
+ expect(batches.length).toBeGreaterThanOrEqual(1);
102
+ expect(batches[0].items).toContain(1);
103
+ });
104
+ it('sessionWindow groups by gap', async () => {
105
+ const items = [
106
+ { v: 1, ts: 0 },
107
+ { v: 2, ts: 10 },
108
+ { v: 3, ts: 100 },
109
+ ];
110
+ async function* gen() {
111
+ for (const x of items)
112
+ yield x;
113
+ }
114
+ const batches = [];
115
+ for await (const b of processor.sessionWindow(gen(), 50, (x) => x.ts)) {
116
+ batches.push(b);
117
+ }
118
+ expect(batches).toHaveLength(2);
119
+ expect(batches[0].items).toHaveLength(2);
120
+ expect(batches[1].items).toHaveLength(1);
121
+ });
122
+ it('joinStreams merges by key', async () => {
123
+ async function* left() {
124
+ yield { id: 'a', name: 'Alice' };
125
+ }
126
+ async function* right() {
127
+ yield { id: 'a', score: 100 };
128
+ }
129
+ const results = [];
130
+ for await (const r of processor.joinStreams(left(), right(), (l) => l.id, (r) => r.id, (l, r) => ({ name: l.name, score: r.score }))) {
131
+ results.push(r);
132
+ }
133
+ expect(results).toHaveLength(1);
134
+ expect(results[0]).toEqual({ name: 'Alice', score: 100 });
135
+ });
136
+ it('joinStreams when right arrives first buffers and matches', async () => {
137
+ async function* left() {
138
+ await new Promise((r) => setTimeout(r, 10));
139
+ yield { id: 'x', leftVal: 1 };
140
+ }
141
+ async function* right() {
142
+ yield { id: 'x', rightVal: 2 };
143
+ }
144
+ const results = [];
145
+ for await (const r of processor.joinStreams(left(), right(), (l) => l.id, (r) => r.id, (l, r) => ({ leftVal: l.leftVal, rightVal: r.rightVal }), 5000)) {
146
+ results.push(r);
147
+ }
148
+ expect(results).toHaveLength(1);
149
+ expect(results[0]).toEqual({ leftVal: 1, rightVal: 2 });
150
+ });
52
151
  });
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=stream.processor.windowing.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"stream.processor.windowing.test.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.windowing.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,69 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const etl_service_1 = require("../pipelines/etl.service");
4
+ const schema_validator_1 = require("../validators/schema.validator");
5
+ const stream_processor_1 = require("./stream.processor");
6
+ async function* itemsWithTs(items) {
7
+ for (const item of items)
8
+ yield item;
9
+ }
10
+ describe('StreamProcessor windowing', () => {
11
+ let processor;
12
+ beforeEach(() => {
13
+ processor = new stream_processor_1.StreamProcessor(new etl_service_1.ETLService(new schema_validator_1.SchemaValidator()));
14
+ });
15
+ it('tumblingWindow groups by time', async () => {
16
+ const source = itemsWithTs([
17
+ { value: 1, timestamp: 0 },
18
+ { value: 2, timestamp: 10 },
19
+ { value: 3, timestamp: 100 },
20
+ ]);
21
+ const batches = [];
22
+ for await (const batch of processor.tumblingWindow(source, 50)) {
23
+ batches.push(batch.items);
24
+ }
25
+ expect(batches.length).toBeGreaterThanOrEqual(1);
26
+ expect(batches.flat()).toContain(1);
27
+ expect(batches.flat()).toContain(2);
28
+ expect(batches.flat()).toContain(3);
29
+ });
30
+ it('sessionWindow groups by gap', async () => {
31
+ const items = [1, 2, 3];
32
+ const getTs = (i) => (i === 2 ? 100 : i);
33
+ const source = (async function* () {
34
+ for (const v of items)
35
+ yield v;
36
+ })();
37
+ const batches = [];
38
+ for await (const batch of processor.sessionWindow(source, 50, getTs)) {
39
+ batches.push(batch.items);
40
+ }
41
+ expect(batches.length).toBeGreaterThanOrEqual(1);
42
+ });
43
+ it('slidingWindow yields overlapping windows', async () => {
44
+ const source = itemsWithTs([
45
+ { value: 1, timestamp: 0 },
46
+ { value: 2, timestamp: 50 },
47
+ { value: 3, timestamp: 100 },
48
+ ]);
49
+ const batches = [];
50
+ for await (const batch of processor.slidingWindow(source, 100, 50)) {
51
+ batches.push(batch.items);
52
+ }
53
+ expect(batches.length).toBeGreaterThanOrEqual(1);
54
+ });
55
+ it('joinStreams merges by key', async () => {
56
+ const left = (async function* () {
57
+ yield { id: 'a', name: 'Alice' };
58
+ })();
59
+ const right = (async function* () {
60
+ yield { id: 'a', score: 100 };
61
+ })();
62
+ const results = [];
63
+ for await (const r of processor.joinStreams(left, right, (l) => l.id, (r) => r.id, (l, r) => ({ name: l.name, score: r.score }))) {
64
+ results.push(r);
65
+ }
66
+ expect(results).toHaveLength(1);
67
+ expect(results[0]).toEqual({ name: 'Alice', score: 100 });
68
+ });
69
+ });
@@ -0,0 +1,124 @@
1
+ /**
2
+ * Telemetry — zero-dependency OpenTelemetry-compatible instrumentation.
3
+ *
4
+ * Works in two modes:
5
+ * 1. **Standalone** — emits structured events to an in-memory log / custom exporters.
6
+ * 2. **OTel** — when @opentelemetry/api is present in the host application,
7
+ * wraps each pipeline step in an OTel span automatically.
8
+ *
9
+ * The package itself does NOT list @opentelemetry/* as dependencies; it uses
10
+ * dynamic optional `require()` so the feature activates transparently when
11
+ * the host already has it installed.
12
+ */
13
+ export interface SpanContext {
14
+ traceId: string;
15
+ spanId: string;
16
+ parentSpanId?: string;
17
+ }
18
+ export interface PipelineSpan {
19
+ traceId: string;
20
+ spanId: string;
21
+ parentSpanId?: string;
22
+ pipeline: string;
23
+ step: number;
24
+ stepName: string;
25
+ startTime: number;
26
+ endTime: number;
27
+ durationMs: number;
28
+ status: 'ok' | 'error';
29
+ error?: string;
30
+ skipped?: boolean;
31
+ attributes: Record<string, string | number | boolean>;
32
+ }
33
+ export interface MetricPoint {
34
+ name: string;
35
+ value: number;
36
+ labels: Record<string, string>;
37
+ timestamp: number;
38
+ }
39
+ export interface LineageEntry {
40
+ traceId: string;
41
+ pipeline: string;
42
+ input: unknown;
43
+ steps: Array<{
44
+ step: number;
45
+ name: string;
46
+ inputHash: string;
47
+ outputHash: string;
48
+ durationMs: number;
49
+ }>;
50
+ output: unknown;
51
+ timestamp: Date;
52
+ }
53
+ export type SpanExporter = (span: PipelineSpan) => void | Promise<void>;
54
+ export type MetricExporter = (metric: MetricPoint) => void | Promise<void>;
55
+ /**
56
+ * TelemetryService — collect spans, metrics, and lineage for pipeline executions.
57
+ *
58
+ * @example
59
+ * const telemetry = TelemetryService.getInstance();
60
+ * telemetry.addSpanExporter((span) => console.log(span));
61
+ * telemetry.enableLineage();
62
+ *
63
+ * // Automatically used by ETLService when registered
64
+ * DataModule.forRoot({ telemetry: { enabled: true, serviceName: 'orders-pipeline' } });
65
+ */
66
+ export declare class TelemetryService {
67
+ private static instance;
68
+ private serviceName;
69
+ private spanExporters;
70
+ private metricExporters;
71
+ private spans;
72
+ private metrics;
73
+ private lineageStore;
74
+ private lineageEnabled;
75
+ private maxSpansInMemory;
76
+ private otelApi;
77
+ constructor(options?: {
78
+ serviceName?: string;
79
+ maxSpansInMemory?: number;
80
+ });
81
+ static getInstance(options?: {
82
+ serviceName?: string;
83
+ }): TelemetryService;
84
+ static reset(): void;
85
+ private tryLoadOtel;
86
+ addSpanExporter(exporter: SpanExporter): this;
87
+ addMetricExporter(exporter: MetricExporter): this;
88
+ enableLineage(): this;
89
+ startTrace(_pipelineName: string): {
90
+ traceId: string;
91
+ rootSpanId: string;
92
+ };
93
+ recordSpan(span: Omit<PipelineSpan, 'traceId' | 'spanId'> & Partial<SpanContext>): Promise<void>;
94
+ recordMetric(name: string, value: number, labels?: Record<string, string>): Promise<void>;
95
+ recordStepMetrics(pipeline: string, stepName: string, durationMs: number, success: boolean, recordCount?: number): Promise<void>;
96
+ startLineage(pipeline: string, input: unknown): LineageEntry;
97
+ recordLineageStep(entry: LineageEntry, step: number, name: string, input: unknown, output: unknown, durationMs: number): void;
98
+ finalizeLineage(entry: LineageEntry, output: unknown): void;
99
+ getSpans(pipeline?: string): PipelineSpan[];
100
+ getMetrics(name?: string): MetricPoint[];
101
+ getLineage(traceId?: string): LineageEntry[];
102
+ /** Compute summary stats for a pipeline across all recorded spans. */
103
+ getSummary(pipeline: string): {
104
+ totalRuns: number;
105
+ successRate: number;
106
+ avgDurationMs: number;
107
+ p95DurationMs: number;
108
+ };
109
+ clear(): void;
110
+ }
111
+ /**
112
+ * Prometheus-format metric exporter factory.
113
+ * Returns a metric exporter that formats points as Prometheus text.
114
+ *
115
+ * @example
116
+ * const { exporter, getText } = createPrometheusExporter();
117
+ * telemetry.addMetricExporter(exporter);
118
+ * // GET /metrics → getText()
119
+ */
120
+ export declare function createPrometheusExporter(): {
121
+ exporter: MetricExporter;
122
+ getText: () => string;
123
+ };
124
+ //# sourceMappingURL=telemetry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/telemetry/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;CACvD;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACxE,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAmB3E;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAiC;IAExD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAS;IAGjC,OAAO,CAAC,OAAO,CAAoC;gBAEvC,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAO;IAM7E,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,gBAAgB;IAOxE,MAAM,CAAC,KAAK,IAAI,IAAI;IAIpB,OAAO,CAAC,WAAW;IAWnB,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAK7C,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAKjD,aAAa,IAAI,IAAI;IAOrB,UAAU,CAAC,aAAa,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMpE,UAAU,CACd,IAAI,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,GAAG,QAAQ,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,GACpE,OAAO,CAAC,IAAI,CAAC;IA+CV,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GAClC,OAAO,CAAC,IAAI,CAAC;IAiBV,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,EAChB,WAAW,SAAI,GACd,OAAO,CAAC,IAAI,CAAC;IAgBhB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,YAAY;IAW5D,iBAAiB,CACf,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,MAAM,GACjB,IAAI;IAWP,eAAe,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI;IAQ3D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE;IAI3C,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE;IAIxC,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE;IAM5C,sEAAsE;IACtE,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACvB;IAmBD,KAAK,IAAI,IAAI;CAKd;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,IAAI;IAC1C,QAAQ,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,MAAM,MAAM,CAAC;CACvB,CA0BA"}