@hazeljs/data 0.2.0-beta.68 → 0.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -61
- package/dist/connectors/connector.interface.d.ts +29 -0
- package/dist/connectors/connector.interface.d.ts.map +1 -0
- package/dist/connectors/connector.interface.js +6 -0
- package/dist/connectors/csv.connector.d.ts +63 -0
- package/dist/connectors/csv.connector.d.ts.map +1 -0
- package/dist/connectors/csv.connector.js +147 -0
- package/dist/connectors/http.connector.d.ts +68 -0
- package/dist/connectors/http.connector.d.ts.map +1 -0
- package/dist/connectors/http.connector.js +131 -0
- package/dist/connectors/index.d.ts +7 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +12 -0
- package/dist/connectors/memory.connector.d.ts +38 -0
- package/dist/connectors/memory.connector.d.ts.map +1 -0
- package/dist/connectors/memory.connector.js +56 -0
- package/dist/connectors/memory.connector.test.d.ts +2 -0
- package/dist/connectors/memory.connector.test.d.ts.map +1 -0
- package/dist/connectors/memory.connector.test.js +43 -0
- package/dist/data.types.d.ts +16 -0
- package/dist/data.types.d.ts.map +1 -1
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.d.ts.map +1 -1
- package/dist/decorators/index.js +8 -1
- package/dist/decorators/pii.decorator.d.ts +59 -0
- package/dist/decorators/pii.decorator.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.js +197 -0
- package/dist/decorators/pii.decorator.test.d.ts +2 -0
- package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.test.js +150 -0
- package/dist/decorators/pipeline.decorator.js +1 -1
- package/dist/decorators/pipeline.decorator.test.js +8 -0
- package/dist/decorators/transform.decorator.d.ts +9 -1
- package/dist/decorators/transform.decorator.d.ts.map +1 -1
- package/dist/decorators/transform.decorator.js +4 -0
- package/dist/decorators/validate.decorator.d.ts +5 -1
- package/dist/decorators/validate.decorator.d.ts.map +1 -1
- package/dist/decorators/validate.decorator.js +4 -0
- package/dist/flink.service.d.ts +30 -0
- package/dist/flink.service.d.ts.map +1 -1
- package/dist/flink.service.js +50 -2
- package/dist/index.d.ts +13 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -8
- package/dist/pipelines/etl.service.d.ts +41 -2
- package/dist/pipelines/etl.service.d.ts.map +1 -1
- package/dist/pipelines/etl.service.js +143 -6
- package/dist/pipelines/etl.service.test.js +215 -0
- package/dist/pipelines/pipeline.builder.d.ts +86 -13
- package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
- package/dist/pipelines/pipeline.builder.js +177 -27
- package/dist/pipelines/pipeline.builder.test.js +160 -12
- package/dist/pipelines/stream.service.test.js +49 -0
- package/dist/quality/quality.service.d.ts +67 -5
- package/dist/quality/quality.service.d.ts.map +1 -1
- package/dist/quality/quality.service.js +259 -20
- package/dist/quality/quality.service.test.js +94 -0
- package/dist/schema/schema.d.ts +92 -12
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +395 -83
- package/dist/schema/schema.test.js +292 -0
- package/dist/streaming/flink/flink.client.d.ts +41 -3
- package/dist/streaming/flink/flink.client.d.ts.map +1 -1
- package/dist/streaming/flink/flink.client.js +171 -8
- package/dist/streaming/flink/flink.client.test.js +2 -2
- package/dist/streaming/flink/flink.job.d.ts +2 -1
- package/dist/streaming/flink/flink.job.d.ts.map +1 -1
- package/dist/streaming/flink/flink.job.js +2 -2
- package/dist/streaming/stream.processor.d.ts +56 -2
- package/dist/streaming/stream.processor.d.ts.map +1 -1
- package/dist/streaming/stream.processor.js +149 -2
- package/dist/streaming/stream.processor.test.js +99 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.windowing.test.js +69 -0
- package/dist/telemetry/telemetry.d.ts +124 -0
- package/dist/telemetry/telemetry.d.ts.map +1 -0
- package/dist/telemetry/telemetry.js +259 -0
- package/dist/telemetry/telemetry.test.d.ts +2 -0
- package/dist/telemetry/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry/telemetry.test.js +51 -0
- package/dist/testing/index.d.ts +12 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/pipeline-test-harness.d.ts +40 -0
- package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.js +55 -0
- package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
- package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.test.js +102 -0
- package/dist/testing/schema-faker.d.ts +32 -0
- package/dist/testing/schema-faker.d.ts.map +1 -0
- package/dist/testing/schema-faker.js +91 -0
- package/dist/testing/schema-faker.test.d.ts +2 -0
- package/dist/testing/schema-faker.test.d.ts.map +1 -0
- package/dist/testing/schema-faker.test.js +66 -0
- package/dist/transformers/built-in.transformers.test.js +28 -0
- package/dist/transformers/transformer.service.test.js +10 -0
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream.processor.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD
|
|
1
|
+
{"version":3,"file":"stream.processor.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGtD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;GASG;AACH,qBAAa,eAAe;IACd,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,EAAE,UAAU;IAE7C,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAIlE,aAAa,CAAC,CAAC,EACpB,gBAAgB,EAAE,MAAM,EACxB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAC7B,cAAc,CAAC,CAAC,CAAC;IAapB;;;;;;OAMG;IACI,cAAc,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,EACtD,QAAQ,EAAE,MAAM,GACf,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IA2BnC;;;;;;OAMG;IACI,aAAa,CAAC,CAAC,EACpB,MAAM,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,EACtD,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACd,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IA6BnC;;;;;;;OAOG;IACI,aAAa,CAAC,CAAC,EACpB,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,EACxB,MAAM,EAAE,MAAM,EACd,YAAY,GAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAyB,GACnD,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAwBnC;;;;;;;;;;OAUG;IACI,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAC1B,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EACtB,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,EACvB,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC5B,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC7B,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,EAC1B,QAAQ,SAAS,GAChB,cAAc,CAAC,GAAG,CAAC;CAuCvB"}
|
|
@@ -6,8 +6,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.StreamProcessor = void 0;
|
|
7
7
|
const core_1 = __importDefault(require("@hazeljs/core"));
|
|
8
8
|
/**
|
|
9
|
-
* Stream Processor
|
|
10
|
-
*
|
|
9
|
+
* Stream Processor — in-process stream processing.
|
|
10
|
+
*
|
|
11
|
+
* Features:
|
|
12
|
+
* - `processItem` / `processStream` — single-item and async-generator streaming
|
|
13
|
+
* - `tumblingWindow` — non-overlapping time windows
|
|
14
|
+
* - `slidingWindow` — overlapping time windows
|
|
15
|
+
* - `sessionWindow` — gap-based grouping (items grouped when gap > idleMs)
|
|
16
|
+
* - `joinStreams` — merge two async iterables by matching key
|
|
11
17
|
*/
|
|
12
18
|
class StreamProcessor {
|
|
13
19
|
constructor(etlService) {
|
|
@@ -27,5 +33,146 @@ class StreamProcessor {
|
|
|
27
33
|
}
|
|
28
34
|
}
|
|
29
35
|
}
|
|
36
|
+
// ─── Windowing ─────────────────────────────────────────────────────────────
|
|
37
|
+
/**
|
|
38
|
+
* Tumbling window — collects items into non-overlapping fixed-size time windows.
|
|
39
|
+
* Each item is assigned to exactly one window.
|
|
40
|
+
*
|
|
41
|
+
* @param source Async iterable of `{ value: T; timestamp: number }` items
|
|
42
|
+
* @param windowMs Window duration in milliseconds
|
|
43
|
+
*/
|
|
44
|
+
async *tumblingWindow(source, windowMs) {
|
|
45
|
+
let windowStart = null;
|
|
46
|
+
let buffer = [];
|
|
47
|
+
for await (const item of source) {
|
|
48
|
+
if (windowStart === null) {
|
|
49
|
+
windowStart = item.timestamp - (item.timestamp % windowMs);
|
|
50
|
+
}
|
|
51
|
+
const itemWindow = item.timestamp - (item.timestamp % windowMs);
|
|
52
|
+
if (itemWindow > windowStart) {
|
|
53
|
+
if (buffer.length > 0) {
|
|
54
|
+
yield { items: buffer, windowStart, windowEnd: windowStart + windowMs };
|
|
55
|
+
}
|
|
56
|
+
windowStart = itemWindow;
|
|
57
|
+
buffer = [];
|
|
58
|
+
}
|
|
59
|
+
buffer.push(item.value);
|
|
60
|
+
}
|
|
61
|
+
if (buffer.length > 0 && windowStart !== null) {
|
|
62
|
+
yield { items: buffer, windowStart, windowEnd: windowStart + windowMs };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Sliding window — each item may appear in multiple windows.
|
|
67
|
+
*
|
|
68
|
+
* @param source Async iterable of `{ value: T; timestamp: number }` items
|
|
69
|
+
* @param windowMs Window size in milliseconds
|
|
70
|
+
* @param slideMs Slide interval in milliseconds (how often a new window starts)
|
|
71
|
+
*/
|
|
72
|
+
async *slidingWindow(source, windowMs, slideMs) {
|
|
73
|
+
const buffer = [];
|
|
74
|
+
const emittedWindows = new Set();
|
|
75
|
+
for await (const item of source) {
|
|
76
|
+
buffer.push(item);
|
|
77
|
+
// Remove items outside the largest possible window
|
|
78
|
+
const oldest = item.timestamp - windowMs;
|
|
79
|
+
while (buffer.length > 0 && buffer[0].timestamp < oldest) {
|
|
80
|
+
buffer.shift();
|
|
81
|
+
}
|
|
82
|
+
// Determine which sliding window this item triggers
|
|
83
|
+
const windowKey = Math.floor(item.timestamp / slideMs) * slideMs;
|
|
84
|
+
if (!emittedWindows.has(windowKey)) {
|
|
85
|
+
emittedWindows.add(windowKey);
|
|
86
|
+
const windowStart = windowKey;
|
|
87
|
+
const windowEnd = windowStart + windowMs;
|
|
88
|
+
const items = buffer
|
|
89
|
+
.filter((b) => b.timestamp >= windowStart && b.timestamp < windowEnd)
|
|
90
|
+
.map((b) => b.value);
|
|
91
|
+
if (items.length > 0) {
|
|
92
|
+
yield { items, windowStart, windowEnd };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Session window — groups items by inactivity gaps.
|
|
99
|
+
* A new window starts when no item is received for longer than `idleMs`.
|
|
100
|
+
*
|
|
101
|
+
* @param source Async iterable of items (with optional `.timestamp`)
|
|
102
|
+
* @param idleMs Gap duration that triggers a new session (default: 30_000ms)
|
|
103
|
+
* @param getTimestamp Function to extract timestamp from an item (default: Date.now())
|
|
104
|
+
*/
|
|
105
|
+
async *sessionWindow(source, idleMs, getTimestamp = () => Date.now()) {
|
|
106
|
+
let buffer = [];
|
|
107
|
+
let lastTimestamp = null;
|
|
108
|
+
let windowStart = null;
|
|
109
|
+
for await (const item of source) {
|
|
110
|
+
const ts = getTimestamp(item);
|
|
111
|
+
if (lastTimestamp !== null && ts - lastTimestamp > idleMs && buffer.length > 0) {
|
|
112
|
+
yield { items: buffer, windowStart: windowStart, windowEnd: lastTimestamp };
|
|
113
|
+
buffer = [];
|
|
114
|
+
windowStart = null;
|
|
115
|
+
}
|
|
116
|
+
if (windowStart === null)
|
|
117
|
+
windowStart = ts;
|
|
118
|
+
buffer.push(item);
|
|
119
|
+
lastTimestamp = ts;
|
|
120
|
+
}
|
|
121
|
+
if (buffer.length > 0 && windowStart !== null && lastTimestamp !== null) {
|
|
122
|
+
yield { items: buffer, windowStart, windowEnd: lastTimestamp };
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Join two async streams by a key function.
|
|
127
|
+
* Items from `left` are buffered; when a matching `right` item arrives, they are emitted together.
|
|
128
|
+
*
|
|
129
|
+
* @param left Left stream
|
|
130
|
+
* @param right Right stream
|
|
131
|
+
* @param leftKey Extract join key from left items
|
|
132
|
+
* @param rightKey Extract join key from right items
|
|
133
|
+
* @param merge Combine matched left + right items
|
|
134
|
+
* @param windowMs How long to buffer unmatched left items (default: 60_000ms)
|
|
135
|
+
*/
|
|
136
|
+
async *joinStreams(left, right, leftKey, rightKey, merge, windowMs = 60000) {
|
|
137
|
+
const leftBuffer = new Map();
|
|
138
|
+
const rightBuffer = new Map();
|
|
139
|
+
const now = () => Date.now();
|
|
140
|
+
// Flatten both streams into a single tagged stream sequentially
|
|
141
|
+
// For a true concurrent join, use a more complex scheduler —
|
|
142
|
+
// this simpler version drains left first, then right, checking buffers on both sides.
|
|
143
|
+
for await (const lItem of left) {
|
|
144
|
+
const key = leftKey(lItem);
|
|
145
|
+
if (rightBuffer.has(key)) {
|
|
146
|
+
const { item: rItem } = rightBuffer.get(key);
|
|
147
|
+
rightBuffer.delete(key);
|
|
148
|
+
yield merge(lItem, rItem);
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
leftBuffer.set(key, { item: lItem, ts: now() });
|
|
152
|
+
}
|
|
153
|
+
// Expire stale unmatched left items
|
|
154
|
+
const expiry = now() - windowMs;
|
|
155
|
+
for (const [k, v] of leftBuffer) {
|
|
156
|
+
if (v.ts < expiry)
|
|
157
|
+
leftBuffer.delete(k);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for await (const rItem of right) {
|
|
161
|
+
const key = rightKey(rItem);
|
|
162
|
+
if (leftBuffer.has(key)) {
|
|
163
|
+
const { item: lItem } = leftBuffer.get(key);
|
|
164
|
+
leftBuffer.delete(key);
|
|
165
|
+
yield merge(lItem, rItem);
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
rightBuffer.set(key, { item: rItem, ts: now() });
|
|
169
|
+
}
|
|
170
|
+
const expiry = now() - windowMs;
|
|
171
|
+
for (const [k, v] of rightBuffer) {
|
|
172
|
+
if (v.ts < expiry)
|
|
173
|
+
rightBuffer.delete(k);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
30
177
|
}
|
|
31
178
|
exports.StreamProcessor = StreamProcessor;
|
|
@@ -49,4 +49,103 @@ describe('StreamProcessor', () => {
|
|
|
49
49
|
}
|
|
50
50
|
expect(results).toEqual([{ v: 2 }, { v: 3 }]);
|
|
51
51
|
});
|
|
52
|
+
it('processStream throws when item fails', async () => {
|
|
53
|
+
let FailPipeline = class FailPipeline {
|
|
54
|
+
fail() {
|
|
55
|
+
throw new Error('Item failed');
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
__decorate([
|
|
59
|
+
(0, decorators_1.Transform)({ step: 1, name: 'fail' }),
|
|
60
|
+
__metadata("design:type", Function),
|
|
61
|
+
__metadata("design:paramtypes", []),
|
|
62
|
+
__metadata("design:returntype", void 0)
|
|
63
|
+
], FailPipeline.prototype, "fail", null);
|
|
64
|
+
FailPipeline = __decorate([
|
|
65
|
+
(0, decorators_1.Pipeline)('fail-proc')
|
|
66
|
+
], FailPipeline);
|
|
67
|
+
async function* gen() {
|
|
68
|
+
yield {};
|
|
69
|
+
}
|
|
70
|
+
await expect((async () => {
|
|
71
|
+
for await (const _ of processor.processStream(new FailPipeline(), gen())) {
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
})()).rejects.toThrow('Item failed');
|
|
75
|
+
});
|
|
76
|
+
it('tumblingWindow groups items by time window', async () => {
|
|
77
|
+
async function* source() {
|
|
78
|
+
yield { value: 1, timestamp: 100 };
|
|
79
|
+
yield { value: 2, timestamp: 150 };
|
|
80
|
+
yield { value: 3, timestamp: 250 };
|
|
81
|
+
}
|
|
82
|
+
const batches = [];
|
|
83
|
+
for await (const b of processor.tumblingWindow(source(), 100)) {
|
|
84
|
+
batches.push({ items: b.items, windowStart: b.windowStart });
|
|
85
|
+
}
|
|
86
|
+
expect(batches).toHaveLength(2);
|
|
87
|
+
expect(batches[0].items).toEqual([1, 2]);
|
|
88
|
+
expect(batches[0].windowStart).toBe(100);
|
|
89
|
+
expect(batches[1].items).toEqual([3]);
|
|
90
|
+
expect(batches[1].windowStart).toBe(200);
|
|
91
|
+
});
|
|
92
|
+
it('slidingWindow emits overlapping windows', async () => {
|
|
93
|
+
async function* source() {
|
|
94
|
+
yield { value: 1, timestamp: 0 };
|
|
95
|
+
yield { value: 2, timestamp: 50 };
|
|
96
|
+
}
|
|
97
|
+
const batches = [];
|
|
98
|
+
for await (const b of processor.slidingWindow(source(), 100, 50)) {
|
|
99
|
+
batches.push(b);
|
|
100
|
+
}
|
|
101
|
+
expect(batches.length).toBeGreaterThanOrEqual(1);
|
|
102
|
+
expect(batches[0].items).toContain(1);
|
|
103
|
+
});
|
|
104
|
+
it('sessionWindow groups by gap', async () => {
|
|
105
|
+
const items = [
|
|
106
|
+
{ v: 1, ts: 0 },
|
|
107
|
+
{ v: 2, ts: 10 },
|
|
108
|
+
{ v: 3, ts: 100 },
|
|
109
|
+
];
|
|
110
|
+
async function* gen() {
|
|
111
|
+
for (const x of items)
|
|
112
|
+
yield x;
|
|
113
|
+
}
|
|
114
|
+
const batches = [];
|
|
115
|
+
for await (const b of processor.sessionWindow(gen(), 50, (x) => x.ts)) {
|
|
116
|
+
batches.push(b);
|
|
117
|
+
}
|
|
118
|
+
expect(batches).toHaveLength(2);
|
|
119
|
+
expect(batches[0].items).toHaveLength(2);
|
|
120
|
+
expect(batches[1].items).toHaveLength(1);
|
|
121
|
+
});
|
|
122
|
+
it('joinStreams merges by key', async () => {
|
|
123
|
+
async function* left() {
|
|
124
|
+
yield { id: 'a', name: 'Alice' };
|
|
125
|
+
}
|
|
126
|
+
async function* right() {
|
|
127
|
+
yield { id: 'a', score: 100 };
|
|
128
|
+
}
|
|
129
|
+
const results = [];
|
|
130
|
+
for await (const r of processor.joinStreams(left(), right(), (l) => l.id, (r) => r.id, (l, r) => ({ name: l.name, score: r.score }))) {
|
|
131
|
+
results.push(r);
|
|
132
|
+
}
|
|
133
|
+
expect(results).toHaveLength(1);
|
|
134
|
+
expect(results[0]).toEqual({ name: 'Alice', score: 100 });
|
|
135
|
+
});
|
|
136
|
+
it('joinStreams when right arrives first buffers and matches', async () => {
|
|
137
|
+
async function* left() {
|
|
138
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
139
|
+
yield { id: 'x', leftVal: 1 };
|
|
140
|
+
}
|
|
141
|
+
async function* right() {
|
|
142
|
+
yield { id: 'x', rightVal: 2 };
|
|
143
|
+
}
|
|
144
|
+
const results = [];
|
|
145
|
+
for await (const r of processor.joinStreams(left(), right(), (l) => l.id, (r) => r.id, (l, r) => ({ leftVal: l.leftVal, rightVal: r.rightVal }), 5000)) {
|
|
146
|
+
results.push(r);
|
|
147
|
+
}
|
|
148
|
+
expect(results).toHaveLength(1);
|
|
149
|
+
expect(results[0]).toEqual({ leftVal: 1, rightVal: 2 });
|
|
150
|
+
});
|
|
52
151
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream.processor.windowing.test.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.processor.windowing.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const etl_service_1 = require("../pipelines/etl.service");
|
|
4
|
+
const schema_validator_1 = require("../validators/schema.validator");
|
|
5
|
+
const stream_processor_1 = require("./stream.processor");
|
|
6
|
+
async function* itemsWithTs(items) {
|
|
7
|
+
for (const item of items)
|
|
8
|
+
yield item;
|
|
9
|
+
}
|
|
10
|
+
describe('StreamProcessor windowing', () => {
|
|
11
|
+
let processor;
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
processor = new stream_processor_1.StreamProcessor(new etl_service_1.ETLService(new schema_validator_1.SchemaValidator()));
|
|
14
|
+
});
|
|
15
|
+
it('tumblingWindow groups by time', async () => {
|
|
16
|
+
const source = itemsWithTs([
|
|
17
|
+
{ value: 1, timestamp: 0 },
|
|
18
|
+
{ value: 2, timestamp: 10 },
|
|
19
|
+
{ value: 3, timestamp: 100 },
|
|
20
|
+
]);
|
|
21
|
+
const batches = [];
|
|
22
|
+
for await (const batch of processor.tumblingWindow(source, 50)) {
|
|
23
|
+
batches.push(batch.items);
|
|
24
|
+
}
|
|
25
|
+
expect(batches.length).toBeGreaterThanOrEqual(1);
|
|
26
|
+
expect(batches.flat()).toContain(1);
|
|
27
|
+
expect(batches.flat()).toContain(2);
|
|
28
|
+
expect(batches.flat()).toContain(3);
|
|
29
|
+
});
|
|
30
|
+
it('sessionWindow groups by gap', async () => {
|
|
31
|
+
const items = [1, 2, 3];
|
|
32
|
+
const getTs = (i) => (i === 2 ? 100 : i);
|
|
33
|
+
const source = (async function* () {
|
|
34
|
+
for (const v of items)
|
|
35
|
+
yield v;
|
|
36
|
+
})();
|
|
37
|
+
const batches = [];
|
|
38
|
+
for await (const batch of processor.sessionWindow(source, 50, getTs)) {
|
|
39
|
+
batches.push(batch.items);
|
|
40
|
+
}
|
|
41
|
+
expect(batches.length).toBeGreaterThanOrEqual(1);
|
|
42
|
+
});
|
|
43
|
+
it('slidingWindow yields overlapping windows', async () => {
|
|
44
|
+
const source = itemsWithTs([
|
|
45
|
+
{ value: 1, timestamp: 0 },
|
|
46
|
+
{ value: 2, timestamp: 50 },
|
|
47
|
+
{ value: 3, timestamp: 100 },
|
|
48
|
+
]);
|
|
49
|
+
const batches = [];
|
|
50
|
+
for await (const batch of processor.slidingWindow(source, 100, 50)) {
|
|
51
|
+
batches.push(batch.items);
|
|
52
|
+
}
|
|
53
|
+
expect(batches.length).toBeGreaterThanOrEqual(1);
|
|
54
|
+
});
|
|
55
|
+
it('joinStreams merges by key', async () => {
|
|
56
|
+
const left = (async function* () {
|
|
57
|
+
yield { id: 'a', name: 'Alice' };
|
|
58
|
+
})();
|
|
59
|
+
const right = (async function* () {
|
|
60
|
+
yield { id: 'a', score: 100 };
|
|
61
|
+
})();
|
|
62
|
+
const results = [];
|
|
63
|
+
for await (const r of processor.joinStreams(left, right, (l) => l.id, (r) => r.id, (l, r) => ({ name: l.name, score: r.score }))) {
|
|
64
|
+
results.push(r);
|
|
65
|
+
}
|
|
66
|
+
expect(results).toHaveLength(1);
|
|
67
|
+
expect(results[0]).toEqual({ name: 'Alice', score: 100 });
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telemetry — zero-dependency OpenTelemetry-compatible instrumentation.
|
|
3
|
+
*
|
|
4
|
+
* Works in two modes:
|
|
5
|
+
* 1. **Standalone** — emits structured events to an in-memory log / custom exporters.
|
|
6
|
+
* 2. **OTel** — when @opentelemetry/api is present in the host application,
|
|
7
|
+
* wraps each pipeline step in an OTel span automatically.
|
|
8
|
+
*
|
|
9
|
+
* The package itself does NOT list @opentelemetry/* as dependencies; it uses
|
|
10
|
+
* dynamic optional `require()` so the feature activates transparently when
|
|
11
|
+
* the host already has it installed.
|
|
12
|
+
*/
|
|
13
|
+
export interface SpanContext {
|
|
14
|
+
traceId: string;
|
|
15
|
+
spanId: string;
|
|
16
|
+
parentSpanId?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface PipelineSpan {
|
|
19
|
+
traceId: string;
|
|
20
|
+
spanId: string;
|
|
21
|
+
parentSpanId?: string;
|
|
22
|
+
pipeline: string;
|
|
23
|
+
step: number;
|
|
24
|
+
stepName: string;
|
|
25
|
+
startTime: number;
|
|
26
|
+
endTime: number;
|
|
27
|
+
durationMs: number;
|
|
28
|
+
status: 'ok' | 'error';
|
|
29
|
+
error?: string;
|
|
30
|
+
skipped?: boolean;
|
|
31
|
+
attributes: Record<string, string | number | boolean>;
|
|
32
|
+
}
|
|
33
|
+
export interface MetricPoint {
|
|
34
|
+
name: string;
|
|
35
|
+
value: number;
|
|
36
|
+
labels: Record<string, string>;
|
|
37
|
+
timestamp: number;
|
|
38
|
+
}
|
|
39
|
+
export interface LineageEntry {
|
|
40
|
+
traceId: string;
|
|
41
|
+
pipeline: string;
|
|
42
|
+
input: unknown;
|
|
43
|
+
steps: Array<{
|
|
44
|
+
step: number;
|
|
45
|
+
name: string;
|
|
46
|
+
inputHash: string;
|
|
47
|
+
outputHash: string;
|
|
48
|
+
durationMs: number;
|
|
49
|
+
}>;
|
|
50
|
+
output: unknown;
|
|
51
|
+
timestamp: Date;
|
|
52
|
+
}
|
|
53
|
+
export type SpanExporter = (span: PipelineSpan) => void | Promise<void>;
|
|
54
|
+
export type MetricExporter = (metric: MetricPoint) => void | Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* TelemetryService — collect spans, metrics, and lineage for pipeline executions.
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* const telemetry = TelemetryService.getInstance();
|
|
60
|
+
* telemetry.addSpanExporter((span) => console.log(span));
|
|
61
|
+
* telemetry.enableLineage();
|
|
62
|
+
*
|
|
63
|
+
* // Automatically used by ETLService when registered
|
|
64
|
+
* DataModule.forRoot({ telemetry: { enabled: true, serviceName: 'orders-pipeline' } });
|
|
65
|
+
*/
|
|
66
|
+
export declare class TelemetryService {
|
|
67
|
+
private static instance;
|
|
68
|
+
private serviceName;
|
|
69
|
+
private spanExporters;
|
|
70
|
+
private metricExporters;
|
|
71
|
+
private spans;
|
|
72
|
+
private metrics;
|
|
73
|
+
private lineageStore;
|
|
74
|
+
private lineageEnabled;
|
|
75
|
+
private maxSpansInMemory;
|
|
76
|
+
private otelApi;
|
|
77
|
+
constructor(options?: {
|
|
78
|
+
serviceName?: string;
|
|
79
|
+
maxSpansInMemory?: number;
|
|
80
|
+
});
|
|
81
|
+
static getInstance(options?: {
|
|
82
|
+
serviceName?: string;
|
|
83
|
+
}): TelemetryService;
|
|
84
|
+
static reset(): void;
|
|
85
|
+
private tryLoadOtel;
|
|
86
|
+
addSpanExporter(exporter: SpanExporter): this;
|
|
87
|
+
addMetricExporter(exporter: MetricExporter): this;
|
|
88
|
+
enableLineage(): this;
|
|
89
|
+
startTrace(_pipelineName: string): {
|
|
90
|
+
traceId: string;
|
|
91
|
+
rootSpanId: string;
|
|
92
|
+
};
|
|
93
|
+
recordSpan(span: Omit<PipelineSpan, 'traceId' | 'spanId'> & Partial<SpanContext>): Promise<void>;
|
|
94
|
+
recordMetric(name: string, value: number, labels?: Record<string, string>): Promise<void>;
|
|
95
|
+
recordStepMetrics(pipeline: string, stepName: string, durationMs: number, success: boolean, recordCount?: number): Promise<void>;
|
|
96
|
+
startLineage(pipeline: string, input: unknown): LineageEntry;
|
|
97
|
+
recordLineageStep(entry: LineageEntry, step: number, name: string, input: unknown, output: unknown, durationMs: number): void;
|
|
98
|
+
finalizeLineage(entry: LineageEntry, output: unknown): void;
|
|
99
|
+
getSpans(pipeline?: string): PipelineSpan[];
|
|
100
|
+
getMetrics(name?: string): MetricPoint[];
|
|
101
|
+
getLineage(traceId?: string): LineageEntry[];
|
|
102
|
+
/** Compute summary stats for a pipeline across all recorded spans. */
|
|
103
|
+
getSummary(pipeline: string): {
|
|
104
|
+
totalRuns: number;
|
|
105
|
+
successRate: number;
|
|
106
|
+
avgDurationMs: number;
|
|
107
|
+
p95DurationMs: number;
|
|
108
|
+
};
|
|
109
|
+
clear(): void;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Prometheus-format metric exporter factory.
|
|
113
|
+
* Returns a metric exporter that formats points as Prometheus text.
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* const { exporter, getText } = createPrometheusExporter();
|
|
117
|
+
* telemetry.addMetricExporter(exporter);
|
|
118
|
+
* // GET /metrics → getText()
|
|
119
|
+
*/
|
|
120
|
+
export declare function createPrometheusExporter(): {
|
|
121
|
+
exporter: MetricExporter;
|
|
122
|
+
getText: () => string;
|
|
123
|
+
};
|
|
124
|
+
//# sourceMappingURL=telemetry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/telemetry/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;CACvD;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACxE,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAmB3E;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAiC;IAExD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAS;IAGjC,OAAO,CAAC,OAAO,CAAoC;gBAEvC,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAO;IAM7E,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,gBAAgB;IAOxE,MAAM,CAAC,KAAK,IAAI,IAAI;IAIpB,OAAO,CAAC,WAAW;IAWnB,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAK7C,iBAAiB,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI;IAKjD,aAAa,IAAI,IAAI;IAOrB,UAAU,CAAC,aAAa,EAAE,MAAM,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMpE,UAAU,CACd,IAAI,EAAE,IAAI,CAAC,YAAY,EAAE,SAAS,GAAG,QAAQ,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,GACpE,OAAO,CAAC,IAAI,CAAC;IA+CV,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GAClC,OAAO,CAAC,IAAI,CAAC;IAiBV,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,OAAO,EAChB,WAAW,SAAI,GACd,OAAO,CAAC,IAAI,CAAC;IAgBhB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,YAAY;IAW5D,iBAAiB,CACf,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,OAAO,EACf,UAAU,EAAE,MAAM,GACjB,IAAI;IAWP,eAAe,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI;IAQ3D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE;IAI3C,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE;IAIxC,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE;IAM5C,sEAAsE;IACtE,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;KACvB;IAmBD,KAAK,IAAI,IAAI;CAKd;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,IAAI;IAC1C,QAAQ,EAAE,cAAc,CAAC;IACzB,OAAO,EAAE,MAAM,MAAM,CAAC;CACvB,CA0BA"}
|