@hazeljs/data 0.2.0-beta.68 → 0.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -61
- package/dist/connectors/connector.interface.d.ts +29 -0
- package/dist/connectors/connector.interface.d.ts.map +1 -0
- package/dist/connectors/connector.interface.js +6 -0
- package/dist/connectors/csv.connector.d.ts +63 -0
- package/dist/connectors/csv.connector.d.ts.map +1 -0
- package/dist/connectors/csv.connector.js +147 -0
- package/dist/connectors/http.connector.d.ts +68 -0
- package/dist/connectors/http.connector.d.ts.map +1 -0
- package/dist/connectors/http.connector.js +131 -0
- package/dist/connectors/index.d.ts +7 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +12 -0
- package/dist/connectors/memory.connector.d.ts +38 -0
- package/dist/connectors/memory.connector.d.ts.map +1 -0
- package/dist/connectors/memory.connector.js +56 -0
- package/dist/connectors/memory.connector.test.d.ts +2 -0
- package/dist/connectors/memory.connector.test.d.ts.map +1 -0
- package/dist/connectors/memory.connector.test.js +43 -0
- package/dist/data.types.d.ts +16 -0
- package/dist/data.types.d.ts.map +1 -1
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.d.ts.map +1 -1
- package/dist/decorators/index.js +8 -1
- package/dist/decorators/pii.decorator.d.ts +59 -0
- package/dist/decorators/pii.decorator.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.js +197 -0
- package/dist/decorators/pii.decorator.test.d.ts +2 -0
- package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.test.js +150 -0
- package/dist/decorators/pipeline.decorator.js +1 -1
- package/dist/decorators/pipeline.decorator.test.js +8 -0
- package/dist/decorators/transform.decorator.d.ts +9 -1
- package/dist/decorators/transform.decorator.d.ts.map +1 -1
- package/dist/decorators/transform.decorator.js +4 -0
- package/dist/decorators/validate.decorator.d.ts +5 -1
- package/dist/decorators/validate.decorator.d.ts.map +1 -1
- package/dist/decorators/validate.decorator.js +4 -0
- package/dist/flink.service.d.ts +30 -0
- package/dist/flink.service.d.ts.map +1 -1
- package/dist/flink.service.js +50 -2
- package/dist/index.d.ts +13 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -8
- package/dist/pipelines/etl.service.d.ts +41 -2
- package/dist/pipelines/etl.service.d.ts.map +1 -1
- package/dist/pipelines/etl.service.js +143 -6
- package/dist/pipelines/etl.service.test.js +215 -0
- package/dist/pipelines/pipeline.builder.d.ts +86 -13
- package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
- package/dist/pipelines/pipeline.builder.js +177 -27
- package/dist/pipelines/pipeline.builder.test.js +160 -12
- package/dist/pipelines/stream.service.test.js +49 -0
- package/dist/quality/quality.service.d.ts +67 -5
- package/dist/quality/quality.service.d.ts.map +1 -1
- package/dist/quality/quality.service.js +259 -20
- package/dist/quality/quality.service.test.js +94 -0
- package/dist/schema/schema.d.ts +92 -12
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +395 -83
- package/dist/schema/schema.test.js +292 -0
- package/dist/streaming/flink/flink.client.d.ts +41 -3
- package/dist/streaming/flink/flink.client.d.ts.map +1 -1
- package/dist/streaming/flink/flink.client.js +171 -8
- package/dist/streaming/flink/flink.client.test.js +2 -2
- package/dist/streaming/flink/flink.job.d.ts +2 -1
- package/dist/streaming/flink/flink.job.d.ts.map +1 -1
- package/dist/streaming/flink/flink.job.js +2 -2
- package/dist/streaming/stream.processor.d.ts +56 -2
- package/dist/streaming/stream.processor.d.ts.map +1 -1
- package/dist/streaming/stream.processor.js +149 -2
- package/dist/streaming/stream.processor.test.js +99 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.windowing.test.js +69 -0
- package/dist/telemetry/telemetry.d.ts +124 -0
- package/dist/telemetry/telemetry.d.ts.map +1 -0
- package/dist/telemetry/telemetry.js +259 -0
- package/dist/telemetry/telemetry.test.d.ts +2 -0
- package/dist/telemetry/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry/telemetry.test.js +51 -0
- package/dist/testing/index.d.ts +12 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/pipeline-test-harness.d.ts +40 -0
- package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.js +55 -0
- package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
- package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.test.js +102 -0
- package/dist/testing/schema-faker.d.ts +32 -0
- package/dist/testing/schema-faker.d.ts.map +1 -0
- package/dist/testing/schema-faker.js +91 -0
- package/dist/testing/schema-faker.test.d.ts +2 -0
- package/dist/testing/schema-faker.test.d.ts.map +1 -0
- package/dist/testing/schema-faker.test.js +66 -0
- package/dist/transformers/built-in.transformers.test.js +28 -0
- package/dist/transformers/transformer.service.test.js +10 -0
- package/package.json +2 -2
|
@@ -116,4 +116,296 @@ describe('Schema', () => {
|
|
|
116
116
|
expect(schema_1.Schema.array(schema_1.Schema.string()).validate({}).success).toBe(false);
|
|
117
117
|
});
|
|
118
118
|
});
|
|
119
|
+
describe('boolean', () => {
|
|
120
|
+
it('validates boolean', () => {
|
|
121
|
+
expect(schema_1.Schema.boolean().validate(true).success).toBe(true);
|
|
122
|
+
expect(schema_1.Schema.boolean().validate(false).success).toBe(true);
|
|
123
|
+
expect(schema_1.Schema.boolean().validate(1).success).toBe(false);
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
describe('literal', () => {
|
|
127
|
+
it('validates literal value', () => {
|
|
128
|
+
expect(schema_1.Schema.literal('active').validate('active').success).toBe(true);
|
|
129
|
+
expect(schema_1.Schema.literal('active').validate('inactive').success).toBe(false);
|
|
130
|
+
expect(schema_1.Schema.literal(42).validate(42).success).toBe(true);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
describe('union', () => {
|
|
134
|
+
it('validates union of schemas', () => {
|
|
135
|
+
const schema = schema_1.Schema.union([schema_1.Schema.literal('a'), schema_1.Schema.literal('b')]);
|
|
136
|
+
expect(schema.validate('a').success).toBe(true);
|
|
137
|
+
expect(schema.validate('b').success).toBe(true);
|
|
138
|
+
expect(schema.validate('c').success).toBe(false);
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
describe('optional and default', () => {
|
|
142
|
+
it('optional accepts undefined', () => {
|
|
143
|
+
const schema = schema_1.Schema.string().optional();
|
|
144
|
+
expect(schema.validate(undefined).success).toBe(true);
|
|
145
|
+
expect(schema.validate('x').success).toBe(true);
|
|
146
|
+
});
|
|
147
|
+
it('default returns default for undefined', () => {
|
|
148
|
+
const schema = schema_1.Schema.string().default('fallback');
|
|
149
|
+
const result = schema.validate(undefined);
|
|
150
|
+
expect(result.success).toBe(true);
|
|
151
|
+
if (result.success)
|
|
152
|
+
expect(result.data).toBe('fallback');
|
|
153
|
+
});
|
|
154
|
+
});
|
|
155
|
+
describe('transform and refine', () => {
|
|
156
|
+
it('transform coerces after validation', () => {
|
|
157
|
+
const schema = schema_1.Schema.number().transform((n) => n * 2);
|
|
158
|
+
const result = schema.validate(21);
|
|
159
|
+
expect(result.success).toBe(true);
|
|
160
|
+
if (result.success)
|
|
161
|
+
expect(result.data).toBe(42);
|
|
162
|
+
});
|
|
163
|
+
it('refine adds custom constraint', () => {
|
|
164
|
+
const schema = schema_1.Schema.number().refine((n) => n % 2 === 0, 'Must be even');
|
|
165
|
+
expect(schema.validate(4).success).toBe(true);
|
|
166
|
+
expect(schema.validate(3).success).toBe(false);
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
describe('toJsonSchema', () => {
|
|
170
|
+
it('exports string schema', () => {
|
|
171
|
+
const js = schema_1.Schema.string().toJsonSchema();
|
|
172
|
+
expect(js).toMatchObject({ type: 'string' });
|
|
173
|
+
});
|
|
174
|
+
it('exports object schema', () => {
|
|
175
|
+
const schema = schema_1.Schema.object({ name: schema_1.Schema.string() });
|
|
176
|
+
const js = schema.toJsonSchema();
|
|
177
|
+
expect(js).toMatchObject({ type: 'object' });
|
|
178
|
+
expect(js).toHaveProperty('properties');
|
|
179
|
+
});
|
|
180
|
+
it('exports array schema', () => {
|
|
181
|
+
const schema = schema_1.Schema.array(schema_1.Schema.number());
|
|
182
|
+
const js = schema.toJsonSchema();
|
|
183
|
+
expect(js).toMatchObject({ type: 'array' });
|
|
184
|
+
expect(js).toHaveProperty('items');
|
|
185
|
+
});
|
|
186
|
+
it('exports literal schema', () => {
|
|
187
|
+
const js = schema_1.Schema.literal('x').toJsonSchema();
|
|
188
|
+
expect(js).toMatchObject({ const: 'x' });
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
describe('nullable', () => {
|
|
192
|
+
it('accepts null', () => {
|
|
193
|
+
const schema = schema_1.Schema.string().nullable();
|
|
194
|
+
expect(schema.validate(null).success).toBe(true);
|
|
195
|
+
expect(schema.validate('x').success).toBe(true);
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
describe('validateAsync', () => {
|
|
199
|
+
it('validates with async refinements', async () => {
|
|
200
|
+
const schema = schema_1.Schema.number().refineAsync(async (n) => n > 0, 'Must be positive');
|
|
201
|
+
const result = await schema.validateAsync(5);
|
|
202
|
+
expect(result.success).toBe(true);
|
|
203
|
+
const fail = await schema.validateAsync(-1);
|
|
204
|
+
expect(fail.success).toBe(false);
|
|
205
|
+
});
|
|
206
|
+
});
|
|
207
|
+
describe('number integer and url', () => {
|
|
208
|
+
it('integer rejects decimals', () => {
|
|
209
|
+
expect(schema_1.Schema.number().integer().validate(5).success).toBe(true);
|
|
210
|
+
expect(schema_1.Schema.number().integer().validate(5.5).success).toBe(false);
|
|
211
|
+
});
|
|
212
|
+
it('string url validates', () => {
|
|
213
|
+
expect(schema_1.Schema.string().url().validate('https://example.com').success).toBe(true);
|
|
214
|
+
expect(schema_1.Schema.string().url().validate('not-a-url').success).toBe(false);
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
describe('object pick omit extend strict', () => {
|
|
218
|
+
const base = schema_1.Schema.object({ a: schema_1.Schema.number(), b: schema_1.Schema.string(), c: schema_1.Schema.boolean() });
|
|
219
|
+
it('pick selects keys', () => {
|
|
220
|
+
const picked = base.pick(['a', 'c']);
|
|
221
|
+
const result = picked.validate({ a: 1, b: 'x', c: true });
|
|
222
|
+
expect(result.success).toBe(true);
|
|
223
|
+
if (result.success) {
|
|
224
|
+
expect(result.data).toEqual({ a: 1, c: true });
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
it('omit removes keys', () => {
|
|
228
|
+
const omitted = base.omit(['b']);
|
|
229
|
+
const result = omitted.validate({ a: 1, b: 'x', c: true });
|
|
230
|
+
expect(result.success).toBe(true);
|
|
231
|
+
if (result.success) {
|
|
232
|
+
expect(result.data).toEqual({ a: 1, c: true });
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
it('extend adds fields', () => {
|
|
236
|
+
const extended = base.extend({ d: schema_1.Schema.number() });
|
|
237
|
+
const result = extended.validate({ a: 1, b: 'x', c: true, d: 4 });
|
|
238
|
+
expect(result.success).toBe(true);
|
|
239
|
+
if (result.success)
|
|
240
|
+
expect(result.data.d).toBe(4);
|
|
241
|
+
});
|
|
242
|
+
it('strict rejects unknown keys', () => {
|
|
243
|
+
const strict = base.strict();
|
|
244
|
+
const result = strict.validate({ a: 1, b: 'x', c: true, extra: 'bad' });
|
|
245
|
+
expect(result.success).toBe(false);
|
|
246
|
+
});
|
|
247
|
+
});
|
|
248
|
+
describe('array min max nonempty', () => {
|
|
249
|
+
it('min rejects short arrays', () => {
|
|
250
|
+
const schema = schema_1.Schema.array(schema_1.Schema.number()).min(2);
|
|
251
|
+
expect(schema.validate([1, 2]).success).toBe(true);
|
|
252
|
+
expect(schema.validate([1]).success).toBe(false);
|
|
253
|
+
});
|
|
254
|
+
it('nonempty rejects empty', () => {
|
|
255
|
+
const schema = schema_1.Schema.array(schema_1.Schema.string()).nonempty();
|
|
256
|
+
expect(schema.validate(['x']).success).toBe(true);
|
|
257
|
+
expect(schema.validate([]).success).toBe(false);
|
|
258
|
+
});
|
|
259
|
+
});
|
|
260
|
+
describe('transform throws', () => {
|
|
261
|
+
it('transform error is caught', () => {
|
|
262
|
+
const schema = schema_1.Schema.number().transform((n) => {
|
|
263
|
+
if (n < 0)
|
|
264
|
+
throw new Error('Negative');
|
|
265
|
+
return n;
|
|
266
|
+
});
|
|
267
|
+
const result = schema.validate(-1);
|
|
268
|
+
expect(result.success).toBe(false);
|
|
269
|
+
if (!result.success)
|
|
270
|
+
expect(result.errors[0].message).toBe('Negative');
|
|
271
|
+
});
|
|
272
|
+
it('transform non-Error throw uses generic message', () => {
|
|
273
|
+
const schema = schema_1.Schema.number().transform(() => {
|
|
274
|
+
throw 'string error';
|
|
275
|
+
});
|
|
276
|
+
const result = schema.validate(1);
|
|
277
|
+
expect(result.success).toBe(false);
|
|
278
|
+
if (!result.success)
|
|
279
|
+
expect(result.errors[0].message).toBe('Transform failed');
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
describe('string pattern required trim', () => {
|
|
283
|
+
it('pattern validates regex', () => {
|
|
284
|
+
expect(schema_1.Schema.string().pattern(/^\d+$/).validate('123').success).toBe(true);
|
|
285
|
+
expect(schema_1.Schema.string().pattern(/^\d+$/, 'Digits only').validate('abc').success).toBe(false);
|
|
286
|
+
});
|
|
287
|
+
it('required rejects empty string', () => {
|
|
288
|
+
expect(schema_1.Schema.string().required().validate('x').success).toBe(true);
|
|
289
|
+
expect(schema_1.Schema.string().required().validate('').success).toBe(false);
|
|
290
|
+
});
|
|
291
|
+
it('trim preprocesses', () => {
|
|
292
|
+
const result = schema_1.Schema.string().trim().validate(' hi ');
|
|
293
|
+
expect(result.success).toBe(true);
|
|
294
|
+
if (result.success)
|
|
295
|
+
expect(result.data).toBe('hi');
|
|
296
|
+
});
|
|
297
|
+
});
|
|
298
|
+
describe('number positive negative multipleOf', () => {
|
|
299
|
+
it('positive rejects zero and negative', () => {
|
|
300
|
+
expect(schema_1.Schema.number().positive().validate(1).success).toBe(true);
|
|
301
|
+
expect(schema_1.Schema.number().positive().validate(0).success).toBe(false);
|
|
302
|
+
expect(schema_1.Schema.number().positive().validate(-1).success).toBe(false);
|
|
303
|
+
});
|
|
304
|
+
it('negative rejects zero and positive', () => {
|
|
305
|
+
expect(schema_1.Schema.number().negative().validate(-1).success).toBe(true);
|
|
306
|
+
expect(schema_1.Schema.number().negative().validate(0).success).toBe(false);
|
|
307
|
+
});
|
|
308
|
+
it('multipleOf validates', () => {
|
|
309
|
+
expect(schema_1.Schema.number().multipleOf(5).validate(10).success).toBe(true);
|
|
310
|
+
expect(schema_1.Schema.number().multipleOf(5).validate(7).success).toBe(false);
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
describe('number and boolean default', () => {
|
|
314
|
+
it('number default for undefined', () => {
|
|
315
|
+
const schema = schema_1.Schema.number().default(42);
|
|
316
|
+
const result = schema.validate(undefined);
|
|
317
|
+
expect(result.success).toBe(true);
|
|
318
|
+
if (result.success)
|
|
319
|
+
expect(result.data).toBe(42);
|
|
320
|
+
});
|
|
321
|
+
it('boolean default for undefined', () => {
|
|
322
|
+
const schema = schema_1.Schema.boolean().default(true);
|
|
323
|
+
const result = schema.validate(undefined);
|
|
324
|
+
expect(result.success).toBe(true);
|
|
325
|
+
if (result.success)
|
|
326
|
+
expect(result.data).toBe(true);
|
|
327
|
+
});
|
|
328
|
+
});
|
|
329
|
+
describe('date min max default', () => {
|
|
330
|
+
it('date min max', () => {
|
|
331
|
+
const min = new Date('2024-01-01');
|
|
332
|
+
const max = new Date('2024-12-31');
|
|
333
|
+
const schema = schema_1.Schema.date().min(min).max(max);
|
|
334
|
+
expect(schema.validate(new Date('2024-06-15')).success).toBe(true);
|
|
335
|
+
expect(schema.validate(new Date('2023-06-15')).success).toBe(false);
|
|
336
|
+
expect(schema.validate(new Date('2025-06-15')).success).toBe(false);
|
|
337
|
+
});
|
|
338
|
+
it('date default for undefined', () => {
|
|
339
|
+
const d = new Date('2024-01-01');
|
|
340
|
+
const schema = schema_1.Schema.date().default(d);
|
|
341
|
+
const result = schema.validate(undefined);
|
|
342
|
+
expect(result.success).toBe(true);
|
|
343
|
+
if (result.success)
|
|
344
|
+
expect(result.data).toEqual(d);
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
describe('nullable toJsonSchema', () => {
|
|
348
|
+
it('nullable string has type array', () => {
|
|
349
|
+
const js = schema_1.Schema.string().nullable().toJsonSchema();
|
|
350
|
+
expect(js.type).toEqual(['string', 'null']);
|
|
351
|
+
});
|
|
352
|
+
it('nullable without base type', () => {
|
|
353
|
+
const schema = schema_1.Schema.literal('x').nullable();
|
|
354
|
+
const js = schema.toJsonSchema();
|
|
355
|
+
expect(js.type).toEqual(['null']);
|
|
356
|
+
});
|
|
357
|
+
});
|
|
358
|
+
describe('optional toJsonSchema', () => {
|
|
359
|
+
it('optional has _optional flag', () => {
|
|
360
|
+
const js = schema_1.Schema.string().optional().toJsonSchema();
|
|
361
|
+
expect(js._optional).toBe(true);
|
|
362
|
+
});
|
|
363
|
+
});
|
|
364
|
+
describe('object pick filters keys not in shape', () => {
|
|
365
|
+
it('pick ignores keys not in shape', () => {
|
|
366
|
+
const base = schema_1.Schema.object({ a: schema_1.Schema.number() });
|
|
367
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
368
|
+
const picked = base.pick(['a', 'nonexistent']);
|
|
369
|
+
const result = picked.validate({ a: 1 });
|
|
370
|
+
expect(result.success).toBe(true);
|
|
371
|
+
if (result.success)
|
|
372
|
+
expect(result.data).toEqual({ a: 1 });
|
|
373
|
+
});
|
|
374
|
+
});
|
|
375
|
+
describe('array max', () => {
|
|
376
|
+
it('max rejects long arrays', () => {
|
|
377
|
+
const schema = schema_1.Schema.array(schema_1.Schema.number()).max(2);
|
|
378
|
+
expect(schema.validate([1, 2]).success).toBe(true);
|
|
379
|
+
expect(schema.validate([1, 2, 3]).success).toBe(false);
|
|
380
|
+
});
|
|
381
|
+
});
|
|
382
|
+
describe('refineAsync', () => {
|
|
383
|
+
it('refineAsync failure path', async () => {
|
|
384
|
+
const schema = schema_1.Schema.number().refineAsync(async (n) => n > 10, 'Must be > 10');
|
|
385
|
+
const result = await schema.validateAsync(5);
|
|
386
|
+
expect(result.success).toBe(false);
|
|
387
|
+
if (!result.success)
|
|
388
|
+
expect(result.errors[0].message).toBe('Must be > 10');
|
|
389
|
+
});
|
|
390
|
+
});
|
|
391
|
+
describe('buildSchema refine failure', () => {
|
|
392
|
+
it('sync refine failure returns error', () => {
|
|
393
|
+
const schema = schema_1.Schema.number().refine((n) => n > 0, 'Must be positive');
|
|
394
|
+
const result = schema.validate(-1);
|
|
395
|
+
expect(result.success).toBe(false);
|
|
396
|
+
if (!result.success)
|
|
397
|
+
expect(result.errors[0].message).toBe('Must be positive');
|
|
398
|
+
});
|
|
399
|
+
});
|
|
400
|
+
describe('validateAsync with sync failure', () => {
|
|
401
|
+
it('returns sync failure without running async refinements', async () => {
|
|
402
|
+
const schema = schema_1.Schema.number()
|
|
403
|
+
.refine((n) => n > 0, 'sync fail')
|
|
404
|
+
.refineAsync(async () => true, 'async');
|
|
405
|
+
const result = await schema.validateAsync(-1);
|
|
406
|
+
expect(result.success).toBe(false);
|
|
407
|
+
if (!result.success)
|
|
408
|
+
expect(result.errors[0].message).toBe('sync fail');
|
|
409
|
+
});
|
|
410
|
+
});
|
|
119
411
|
});
|
|
@@ -48,11 +48,49 @@ export declare class FlinkClient {
|
|
|
48
48
|
'slots-total'?: number;
|
|
49
49
|
}>;
|
|
50
50
|
getTaskManagers(): Promise<unknown[]>;
|
|
51
|
+
/**
|
|
52
|
+
* Upload a JAR file to the Flink cluster.
|
|
53
|
+
* @param jarPath Local filesystem path to the JAR file
|
|
54
|
+
* @returns The Flink JAR ID for subsequent run calls
|
|
55
|
+
*/
|
|
56
|
+
uploadJar(jarPath: string): Promise<string>;
|
|
57
|
+
/**
|
|
58
|
+
* Run a previously uploaded JAR as a Flink job.
|
|
59
|
+
* @param jarId The JAR ID returned by uploadJar()
|
|
60
|
+
* @param request Optional run parameters (entry class, parallelism, args, etc.)
|
|
61
|
+
* @returns The Flink job ID
|
|
62
|
+
*/
|
|
63
|
+
runJar(jarId: string, request?: FlinkJobSubmitRequest): Promise<string>;
|
|
64
|
+
/**
|
|
65
|
+
* Submit a SQL statement to the Flink SQL Gateway.
|
|
66
|
+
* Requires Flink SQL Gateway to be running.
|
|
67
|
+
* @param sql The SQL statement to execute (CREATE TABLE, INSERT INTO, etc.)
|
|
68
|
+
* @param sessionId An existing Flink SQL Gateway session ID
|
|
69
|
+
*/
|
|
70
|
+
submitSql(sql: string, sessionId: string): Promise<string>;
|
|
71
|
+
/**
|
|
72
|
+
* Create a new SQL Gateway session.
|
|
73
|
+
* @returns The session ID
|
|
74
|
+
*/
|
|
75
|
+
createSqlSession(properties?: Record<string, string>): Promise<string>;
|
|
51
76
|
/**
|
|
52
77
|
* Submit a job to Flink cluster.
|
|
53
|
-
*
|
|
54
|
-
*
|
|
78
|
+
* Requires `jarFile` path in the jobConfig or uses FlinkJobSubmitRequest.jarFile.
|
|
79
|
+
*
|
|
80
|
+
* Workflow: upload JAR → run JAR → return job ID.
|
|
81
|
+
*/
|
|
82
|
+
submitJob(jobConfig: FlinkJobConfig, request?: FlinkJobSubmitRequest): Promise<string>;
|
|
83
|
+
/**
|
|
84
|
+
* List uploaded JARs on the cluster.
|
|
85
|
+
*/
|
|
86
|
+
listJars(): Promise<Array<{
|
|
87
|
+
id: string;
|
|
88
|
+
name: string;
|
|
89
|
+
uploaded: number;
|
|
90
|
+
}>>;
|
|
91
|
+
/**
|
|
92
|
+
* Delete an uploaded JAR from the cluster.
|
|
55
93
|
*/
|
|
56
|
-
|
|
94
|
+
deleteJar(jarId: string): Promise<void>;
|
|
57
95
|
}
|
|
58
96
|
//# sourceMappingURL=flink.client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flink.client.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,qBAAa,WAAW;IACtB,SAAgB,GAAG,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;gBAE/B,MAAM,EAAE,iBAAiB;YASvB,OAAO;IA4Cf,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IA+BnC,YAAY,CAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAY9D,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAKjF,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAKzF,cAAc,IAAI,OAAO,CAAC;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI5E,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAK3C;;;;OAIG;IACG,SAAS,CAAC,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"flink.client.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,qBAAa,WAAW;IACtB,SAAgB,GAAG,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;gBAE/B,MAAM,EAAE,iBAAiB;YASvB,OAAO;IA4Cf,QAAQ,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IA+BnC,YAAY,CAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAY9D,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAKjF,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAKzF,cAAc,IAAI,OAAO,CAAC;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI5E,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAK3C;;;;OAIG;IACG,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA8DjD;;;;;OAKG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,MAAM,CAAC;IAejF;;;;;OAKG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWhE;;;OAGG;IACG,gBAAgB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAW5E;;;;;OAKG;IACG,SAAS,CAAC,SAAS,EAAE,cAAc,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBhG;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAOhF;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAG9C"}
|
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.FlinkClient = void 0;
|
|
4
37
|
/**
|
|
@@ -88,17 +121,147 @@ class FlinkClient {
|
|
|
88
121
|
const result = await this.request('GET', '/taskmanagers');
|
|
89
122
|
return result.taskmanagers ?? [];
|
|
90
123
|
}
|
|
124
|
+
/**
|
|
125
|
+
* Upload a JAR file to the Flink cluster.
|
|
126
|
+
* @param jarPath Local filesystem path to the JAR file
|
|
127
|
+
* @returns The Flink JAR ID for subsequent run calls
|
|
128
|
+
*/
|
|
129
|
+
async uploadJar(jarPath) {
|
|
130
|
+
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
|
|
131
|
+
const path = await Promise.resolve().then(() => __importStar(require('path')));
|
|
132
|
+
if (!fs.existsSync(jarPath)) {
|
|
133
|
+
throw new Error(`JAR file not found: ${jarPath}`);
|
|
134
|
+
}
|
|
135
|
+
const url = `${this.url}/jars/upload`;
|
|
136
|
+
const fileName = path.basename(jarPath);
|
|
137
|
+
const fileBuffer = fs.readFileSync(jarPath);
|
|
138
|
+
const headers = {};
|
|
139
|
+
if (this.config.auth?.type === 'basic' && this.config.auth.username) {
|
|
140
|
+
headers['Authorization'] =
|
|
141
|
+
`Basic ${Buffer.from(`${this.config.auth.username}:${this.config.auth.password ?? ''}`).toString('base64')}`;
|
|
142
|
+
}
|
|
143
|
+
else if (this.config.auth?.type === 'token' && this.config.auth.token) {
|
|
144
|
+
headers['Authorization'] = `Bearer ${this.config.auth.token}`;
|
|
145
|
+
}
|
|
146
|
+
const boundary = `----HazelJSFormBoundary${Date.now()}`;
|
|
147
|
+
const bodyParts = [
|
|
148
|
+
Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="jarfile"; filename="${fileName}"\r\nContent-Type: application/java-archive\r\n\r\n`),
|
|
149
|
+
fileBuffer,
|
|
150
|
+
Buffer.from(`\r\n--${boundary}--\r\n`),
|
|
151
|
+
];
|
|
152
|
+
const body = Buffer.concat(bodyParts);
|
|
153
|
+
const controller = new AbortController();
|
|
154
|
+
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
|
|
155
|
+
try {
|
|
156
|
+
const response = await fetch(url, {
|
|
157
|
+
method: 'POST',
|
|
158
|
+
headers: {
|
|
159
|
+
...headers,
|
|
160
|
+
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
|
161
|
+
'Content-Length': String(body.length),
|
|
162
|
+
},
|
|
163
|
+
body,
|
|
164
|
+
signal: controller.signal,
|
|
165
|
+
});
|
|
166
|
+
clearTimeout(timeoutId);
|
|
167
|
+
if (!response.ok) {
|
|
168
|
+
throw new Error(`Flink JAR upload failed ${response.status}: ${await response.text()}`);
|
|
169
|
+
}
|
|
170
|
+
const result = (await response.json());
|
|
171
|
+
const filename = result.filename ?? '';
|
|
172
|
+
const match = /\/jars\/([^/]+)$/.exec(filename);
|
|
173
|
+
if (!match)
|
|
174
|
+
throw new Error(`Could not extract JAR ID from Flink response: ${filename}`);
|
|
175
|
+
return match[1];
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
clearTimeout(timeoutId);
|
|
179
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Run a previously uploaded JAR as a Flink job.
|
|
184
|
+
* @param jarId The JAR ID returned by uploadJar()
|
|
185
|
+
* @param request Optional run parameters (entry class, parallelism, args, etc.)
|
|
186
|
+
* @returns The Flink job ID
|
|
187
|
+
*/
|
|
188
|
+
async runJar(jarId, request = {}) {
|
|
189
|
+
const body = {};
|
|
190
|
+
if (request.jobName)
|
|
191
|
+
body['job-name'] = request.jobName;
|
|
192
|
+
if (request.parallelism)
|
|
193
|
+
body['parallelism'] = request.parallelism;
|
|
194
|
+
if (request.entryClass)
|
|
195
|
+
body['entry-class'] = request.entryClass;
|
|
196
|
+
if (request.programArgs)
|
|
197
|
+
body['program-args'] = request.programArgs;
|
|
198
|
+
if (request.savepointPath)
|
|
199
|
+
body['savepointPath'] = request.savepointPath;
|
|
200
|
+
if (request.allowNonRestoredState)
|
|
201
|
+
body['allowNonRestoredState'] = request.allowNonRestoredState;
|
|
202
|
+
const result = await this.request('POST', `/jars/${jarId}/run`, body);
|
|
203
|
+
if (!result.jobid)
|
|
204
|
+
throw new Error('Flink did not return a job ID');
|
|
205
|
+
return result.jobid;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Submit a SQL statement to the Flink SQL Gateway.
|
|
209
|
+
* Requires Flink SQL Gateway to be running.
|
|
210
|
+
* @param sql The SQL statement to execute (CREATE TABLE, INSERT INTO, etc.)
|
|
211
|
+
* @param sessionId An existing Flink SQL Gateway session ID
|
|
212
|
+
*/
|
|
213
|
+
async submitSql(sql, sessionId) {
|
|
214
|
+
const result = await this.request('POST', `/sessions/${sessionId}/statements`, { statement: sql });
|
|
215
|
+
const id = result.operationHandle?.identifier;
|
|
216
|
+
if (!id)
|
|
217
|
+
throw new Error('Flink SQL Gateway did not return an operation handle');
|
|
218
|
+
return id;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Create a new SQL Gateway session.
|
|
222
|
+
* @returns The session ID
|
|
223
|
+
*/
|
|
224
|
+
async createSqlSession(properties) {
|
|
225
|
+
const result = await this.request('POST', '/sessions', { properties: properties ?? {} });
|
|
226
|
+
const id = result.sessionHandle?.identifier;
|
|
227
|
+
if (!id)
|
|
228
|
+
throw new Error('Flink SQL Gateway did not return a session handle');
|
|
229
|
+
return id;
|
|
230
|
+
}
|
|
91
231
|
/**
|
|
92
232
|
* Submit a job to Flink cluster.
|
|
93
|
-
*
|
|
94
|
-
*
|
|
233
|
+
* Requires `jarFile` path in the jobConfig or uses FlinkJobSubmitRequest.jarFile.
|
|
234
|
+
*
|
|
235
|
+
* Workflow: upload JAR → run JAR → return job ID.
|
|
236
|
+
*/
|
|
237
|
+
async submitJob(jobConfig, request = {}) {
|
|
238
|
+
const jarFile = request.jarFile;
|
|
239
|
+
if (!jarFile) {
|
|
240
|
+
throw new Error('submitJob requires a JAR file path. Set request.jarFile to the path of your pipeline JAR, ' +
|
|
241
|
+
'or use submitSql() for Flink SQL-based pipelines.');
|
|
242
|
+
}
|
|
243
|
+
const jarId = await this.uploadJar(jarFile);
|
|
244
|
+
return this.runJar(jarId, {
|
|
245
|
+
jobName: request.jobName ?? jobConfig.jobName,
|
|
246
|
+
parallelism: request.parallelism ?? jobConfig.parallelism,
|
|
247
|
+
entryClass: request.entryClass,
|
|
248
|
+
programArgs: request.programArgs,
|
|
249
|
+
savepointPath: request.savepointPath,
|
|
250
|
+
allowNonRestoredState: request.allowNonRestoredState,
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* List uploaded JARs on the cluster.
|
|
255
|
+
*/
|
|
256
|
+
async listJars() {
|
|
257
|
+
const result = await this.request('GET', '/jars');
|
|
258
|
+
return result.files ?? [];
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Delete an uploaded JAR from the cluster.
|
|
95
262
|
*/
|
|
96
|
-
async
|
|
97
|
-
|
|
98
|
-
// For now return a placeholder - full implementation would:
|
|
99
|
-
// 1. Upload JAR to /jars
|
|
100
|
-
// 2. Create job from JAR via /jars/:jarid/run
|
|
101
|
-
throw new Error('submitJob: Full Flink deployment requires JAR submission. Use FlinkService.deployStream() for pipeline-to-job conversion.');
|
|
263
|
+
async deleteJar(jarId) {
|
|
264
|
+
await this.request('DELETE', `/jars/${jarId}`);
|
|
102
265
|
}
|
|
103
266
|
}
|
|
104
267
|
exports.FlinkClient = FlinkClient;
|
|
@@ -40,8 +40,8 @@ describe('FlinkClient', () => {
|
|
|
40
40
|
mockFetch.mockResolvedValue({ ok: false, text: () => Promise.resolve('Not found') });
|
|
41
41
|
await expect(client.getJobStatus('bad')).rejects.toThrow('Flink API error');
|
|
42
42
|
});
|
|
43
|
-
it('submitJob
|
|
44
|
-
await expect(client.submitJob({})).rejects.toThrow('submitJob
|
|
43
|
+
it('submitJob requires a jarFile path', async () => {
|
|
44
|
+
await expect(client.submitJob({})).rejects.toThrow('submitJob requires a JAR file path');
|
|
45
45
|
});
|
|
46
46
|
it('getClusterInfo', async () => {
|
|
47
47
|
mockFetch.mockResolvedValue({
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { FlinkClient } from './flink.client';
|
|
2
2
|
import type { FlinkJobConfig } from '../../data.types';
|
|
3
|
+
import type { FlinkJobSubmitRequest } from './flink.client';
|
|
3
4
|
export interface FlinkJobResult {
|
|
4
5
|
jobId: string;
|
|
5
6
|
status: string;
|
|
@@ -23,6 +24,6 @@ export declare class FlinkJob {
|
|
|
23
24
|
stop(jobId: string, savepointPath?: string): Promise<{
|
|
24
25
|
'request-id': string;
|
|
25
26
|
}>;
|
|
26
|
-
submit(config: FlinkJobConfig,
|
|
27
|
+
submit(config: FlinkJobConfig, request?: FlinkJobSubmitRequest): Promise<string>;
|
|
27
28
|
}
|
|
28
29
|
//# sourceMappingURL=flink.job.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flink.job.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"flink.job.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,WAAW;IAE1C,SAAS,CACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9D,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIzF,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9E,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;CAGvF"}
|
|
@@ -20,8 +20,8 @@ class FlinkJob {
|
|
|
20
20
|
async stop(jobId, savepointPath) {
|
|
21
21
|
return this.client.stopJob(jobId, savepointPath);
|
|
22
22
|
}
|
|
23
|
-
async submit(config,
|
|
24
|
-
return this.client.submitJob(config,
|
|
23
|
+
async submit(config, request) {
|
|
24
|
+
return this.client.submitJob(config, request);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
exports.FlinkJob = FlinkJob;
|
|
@@ -1,12 +1,66 @@
|
|
|
1
1
|
import { ETLService } from '../pipelines/etl.service';
|
|
2
|
+
export interface WindowedBatch<T> {
|
|
3
|
+
items: T[];
|
|
4
|
+
windowStart: number;
|
|
5
|
+
windowEnd: number;
|
|
6
|
+
}
|
|
2
7
|
/**
|
|
3
|
-
* Stream Processor
|
|
4
|
-
*
|
|
8
|
+
* Stream Processor — in-process stream processing.
|
|
9
|
+
*
|
|
10
|
+
* Features:
|
|
11
|
+
* - `processItem` / `processStream` — single-item and async-generator streaming
|
|
12
|
+
* - `tumblingWindow` — non-overlapping time windows
|
|
13
|
+
* - `slidingWindow` — overlapping time windows
|
|
14
|
+
* - `sessionWindow` — gap-based grouping (items grouped when gap > idleMs)
|
|
15
|
+
* - `joinStreams` — merge two async iterables by matching key
|
|
5
16
|
*/
|
|
6
17
|
export declare class StreamProcessor {
|
|
7
18
|
private readonly etlService;
|
|
8
19
|
constructor(etlService: ETLService);
|
|
9
20
|
processItem<T>(pipelineInstance: object, item: unknown): Promise<T>;
|
|
10
21
|
processStream<T>(pipelineInstance: object, source: AsyncIterable<unknown>): AsyncGenerator<T>;
|
|
22
|
+
/**
|
|
23
|
+
* Tumbling window — collects items into non-overlapping fixed-size time windows.
|
|
24
|
+
* Each item is assigned to exactly one window.
|
|
25
|
+
*
|
|
26
|
+
* @param source Async iterable of `{ value: T; timestamp: number }` items
|
|
27
|
+
* @param windowMs Window duration in milliseconds
|
|
28
|
+
*/
|
|
29
|
+
tumblingWindow<T>(source: AsyncIterable<{
|
|
30
|
+
value: T;
|
|
31
|
+
timestamp: number;
|
|
32
|
+
}>, windowMs: number): AsyncGenerator<WindowedBatch<T>>;
|
|
33
|
+
/**
|
|
34
|
+
* Sliding window — each item may appear in multiple windows.
|
|
35
|
+
*
|
|
36
|
+
* @param source Async iterable of `{ value: T; timestamp: number }` items
|
|
37
|
+
* @param windowMs Window size in milliseconds
|
|
38
|
+
* @param slideMs Slide interval in milliseconds (how often a new window starts)
|
|
39
|
+
*/
|
|
40
|
+
slidingWindow<T>(source: AsyncIterable<{
|
|
41
|
+
value: T;
|
|
42
|
+
timestamp: number;
|
|
43
|
+
}>, windowMs: number, slideMs: number): AsyncGenerator<WindowedBatch<T>>;
|
|
44
|
+
/**
|
|
45
|
+
* Session window — groups items by inactivity gaps.
|
|
46
|
+
* A new window starts when no item is received for longer than `idleMs`.
|
|
47
|
+
*
|
|
48
|
+
* @param source Async iterable of items (with optional `.timestamp`)
|
|
49
|
+
* @param idleMs Gap duration that triggers a new session (default: 30_000ms)
|
|
50
|
+
* @param getTimestamp Function to extract timestamp from an item (default: Date.now())
|
|
51
|
+
*/
|
|
52
|
+
sessionWindow<T>(source: AsyncIterable<T>, idleMs: number, getTimestamp?: (item: T) => number): AsyncGenerator<WindowedBatch<T>>;
|
|
53
|
+
/**
|
|
54
|
+
* Join two async streams by a key function.
|
|
55
|
+
* Items from `left` are buffered; when a matching `right` item arrives, they are emitted together.
|
|
56
|
+
*
|
|
57
|
+
* @param left Left stream
|
|
58
|
+
* @param right Right stream
|
|
59
|
+
* @param leftKey Extract join key from left items
|
|
60
|
+
* @param rightKey Extract join key from right items
|
|
61
|
+
* @param merge Combine matched left + right items
|
|
62
|
+
* @param windowMs How long to buffer unmatched left items (default: 60_000ms)
|
|
63
|
+
*/
|
|
64
|
+
joinStreams<L, R, Out>(left: AsyncIterable<L>, right: AsyncIterable<R>, leftKey: (item: L) => string, rightKey: (item: R) => string, merge: (l: L, r: R) => Out, windowMs?: number): AsyncGenerator<Out>;
|
|
11
65
|
}
|
|
12
66
|
//# sourceMappingURL=stream.processor.d.ts.map
|