@hazeljs/data 0.2.0-beta.67 → 0.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -61
- package/dist/connectors/connector.interface.d.ts +29 -0
- package/dist/connectors/connector.interface.d.ts.map +1 -0
- package/dist/connectors/connector.interface.js +6 -0
- package/dist/connectors/csv.connector.d.ts +63 -0
- package/dist/connectors/csv.connector.d.ts.map +1 -0
- package/dist/connectors/csv.connector.js +147 -0
- package/dist/connectors/http.connector.d.ts +68 -0
- package/dist/connectors/http.connector.d.ts.map +1 -0
- package/dist/connectors/http.connector.js +131 -0
- package/dist/connectors/index.d.ts +7 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +12 -0
- package/dist/connectors/memory.connector.d.ts +38 -0
- package/dist/connectors/memory.connector.d.ts.map +1 -0
- package/dist/connectors/memory.connector.js +56 -0
- package/dist/connectors/memory.connector.test.d.ts +2 -0
- package/dist/connectors/memory.connector.test.d.ts.map +1 -0
- package/dist/connectors/memory.connector.test.js +43 -0
- package/dist/data.types.d.ts +16 -0
- package/dist/data.types.d.ts.map +1 -1
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.d.ts.map +1 -1
- package/dist/decorators/index.js +8 -1
- package/dist/decorators/pii.decorator.d.ts +59 -0
- package/dist/decorators/pii.decorator.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.js +197 -0
- package/dist/decorators/pii.decorator.test.d.ts +2 -0
- package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.test.js +150 -0
- package/dist/decorators/pipeline.decorator.js +1 -1
- package/dist/decorators/pipeline.decorator.test.js +8 -0
- package/dist/decorators/transform.decorator.d.ts +9 -1
- package/dist/decorators/transform.decorator.d.ts.map +1 -1
- package/dist/decorators/transform.decorator.js +4 -0
- package/dist/decorators/validate.decorator.d.ts +5 -1
- package/dist/decorators/validate.decorator.d.ts.map +1 -1
- package/dist/decorators/validate.decorator.js +4 -0
- package/dist/flink.service.d.ts +30 -0
- package/dist/flink.service.d.ts.map +1 -1
- package/dist/flink.service.js +50 -2
- package/dist/index.d.ts +13 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -8
- package/dist/pipelines/etl.service.d.ts +41 -2
- package/dist/pipelines/etl.service.d.ts.map +1 -1
- package/dist/pipelines/etl.service.js +143 -6
- package/dist/pipelines/etl.service.test.js +215 -0
- package/dist/pipelines/pipeline.builder.d.ts +86 -13
- package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
- package/dist/pipelines/pipeline.builder.js +177 -27
- package/dist/pipelines/pipeline.builder.test.js +160 -12
- package/dist/pipelines/stream.service.test.js +49 -0
- package/dist/quality/quality.service.d.ts +67 -5
- package/dist/quality/quality.service.d.ts.map +1 -1
- package/dist/quality/quality.service.js +259 -20
- package/dist/quality/quality.service.test.js +94 -0
- package/dist/schema/schema.d.ts +92 -12
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +395 -83
- package/dist/schema/schema.test.js +292 -0
- package/dist/streaming/flink/flink.client.d.ts +41 -3
- package/dist/streaming/flink/flink.client.d.ts.map +1 -1
- package/dist/streaming/flink/flink.client.js +171 -8
- package/dist/streaming/flink/flink.client.test.js +2 -2
- package/dist/streaming/flink/flink.job.d.ts +2 -1
- package/dist/streaming/flink/flink.job.d.ts.map +1 -1
- package/dist/streaming/flink/flink.job.js +2 -2
- package/dist/streaming/stream.processor.d.ts +56 -2
- package/dist/streaming/stream.processor.d.ts.map +1 -1
- package/dist/streaming/stream.processor.js +149 -2
- package/dist/streaming/stream.processor.test.js +99 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.windowing.test.js +69 -0
- package/dist/telemetry/telemetry.d.ts +124 -0
- package/dist/telemetry/telemetry.d.ts.map +1 -0
- package/dist/telemetry/telemetry.js +259 -0
- package/dist/telemetry/telemetry.test.d.ts +2 -0
- package/dist/telemetry/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry/telemetry.test.js +51 -0
- package/dist/testing/index.d.ts +12 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/pipeline-test-harness.d.ts +40 -0
- package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.js +55 -0
- package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
- package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.test.js +102 -0
- package/dist/testing/schema-faker.d.ts +32 -0
- package/dist/testing/schema-faker.d.ts.map +1 -0
- package/dist/testing/schema-faker.js +91 -0
- package/dist/testing/schema-faker.test.d.ts +2 -0
- package/dist/testing/schema-faker.test.d.ts.map +1 -0
- package/dist/testing/schema-faker.test.js +66 -0
- package/dist/transformers/built-in.transformers.test.js +28 -0
- package/dist/transformers/transformer.service.test.js +10 -0
- package/package.json +2 -2
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SchemaFaker = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Generate fake data that matches a schema.
|
|
6
|
+
* Uses schema.toJsonSchema() to infer structure and produce valid sample data.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* const UserSchema = Schema.object({ name: Schema.string(), age: Schema.number() });
|
|
10
|
+
* const fake = SchemaFaker.generate(UserSchema);
|
|
11
|
+
* // { name: "random-string-123", age: 42 }
|
|
12
|
+
*/
|
|
13
|
+
class SchemaFaker {
|
|
14
|
+
constructor(options = {}) {
|
|
15
|
+
this.options = { arrayMinLength: 1, arrayMaxLength: 5, ...options };
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Generate one fake record matching the schema.
|
|
19
|
+
*/
|
|
20
|
+
generate(schema) {
|
|
21
|
+
const js = schema.toJsonSchema();
|
|
22
|
+
return this.generateFromJsonSchema(js);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Generate N fake records.
|
|
26
|
+
*/
|
|
27
|
+
generateMany(schema, count) {
|
|
28
|
+
return Array.from({ length: count }, () => this.generate(schema));
|
|
29
|
+
}
|
|
30
|
+
generateFromJsonSchema(js) {
|
|
31
|
+
if (js.const !== undefined)
|
|
32
|
+
return js.const;
|
|
33
|
+
if (js.oneOf) {
|
|
34
|
+
const schemas = js.oneOf;
|
|
35
|
+
const idx = Math.floor(Math.random() * schemas.length);
|
|
36
|
+
return this.generateFromJsonSchema(schemas[idx]);
|
|
37
|
+
}
|
|
38
|
+
if (js.properties) {
|
|
39
|
+
const out = {};
|
|
40
|
+
const props = js.properties;
|
|
41
|
+
for (const [k, v] of Object.entries(props)) {
|
|
42
|
+
out[k] = this.generateFromJsonSchema(v);
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
if (js.items) {
|
|
47
|
+
const len = Math.floor(Math.random() * (this.options.arrayMaxLength - this.options.arrayMinLength + 1)) + this.options.arrayMinLength;
|
|
48
|
+
const itemSchema = js.items;
|
|
49
|
+
return Array.from({ length: len }, () => this.generateFromJsonSchema(itemSchema));
|
|
50
|
+
}
|
|
51
|
+
const type = js.type;
|
|
52
|
+
if (Array.isArray(type)) {
|
|
53
|
+
const t = type.find((x) => x !== 'null') ?? 'string';
|
|
54
|
+
return this.genByType(t, js);
|
|
55
|
+
}
|
|
56
|
+
if (typeof type === 'string') {
|
|
57
|
+
return this.genByType(type, js);
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
genByType(type, _js) {
|
|
62
|
+
switch (type) {
|
|
63
|
+
case 'string':
|
|
64
|
+
return this.randomString();
|
|
65
|
+
case 'number':
|
|
66
|
+
case 'integer':
|
|
67
|
+
return Math.floor(Math.random() * 1000) - 100;
|
|
68
|
+
case 'boolean':
|
|
69
|
+
return Math.random() > 0.5;
|
|
70
|
+
case 'object':
|
|
71
|
+
return {};
|
|
72
|
+
case 'array':
|
|
73
|
+
return [];
|
|
74
|
+
default:
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
randomString() {
|
|
79
|
+
const chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
|
|
80
|
+
const len = Math.floor(Math.random() * 10) + 5;
|
|
81
|
+
return Array.from({ length: len }, () => chars[Math.floor(Math.random() * chars.length)]).join('');
|
|
82
|
+
}
|
|
83
|
+
/** Static convenience method */
|
|
84
|
+
static generate(schema) {
|
|
85
|
+
return new SchemaFaker().generate(schema);
|
|
86
|
+
}
|
|
87
|
+
static generateMany(schema, count) {
|
|
88
|
+
return new SchemaFaker().generateMany(schema, count);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
exports.SchemaFaker = SchemaFaker;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-faker.test.d.ts","sourceRoot":"","sources":["../../src/testing/schema-faker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const schema_1 = require("../schema/schema");
|
|
4
|
+
const schema_faker_1 = require("./schema-faker");
|
|
5
|
+
describe('SchemaFaker', () => {
|
|
6
|
+
it('generates string', () => {
|
|
7
|
+
const schema = schema_1.Schema.string();
|
|
8
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
9
|
+
expect(typeof value).toBe('string');
|
|
10
|
+
expect(schema.validate(value).success).toBe(true);
|
|
11
|
+
});
|
|
12
|
+
it('generates number', () => {
|
|
13
|
+
const schema = schema_1.Schema.number();
|
|
14
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
15
|
+
expect(typeof value).toBe('number');
|
|
16
|
+
expect(schema.validate(value).success).toBe(true);
|
|
17
|
+
});
|
|
18
|
+
it('generates boolean', () => {
|
|
19
|
+
const schema = schema_1.Schema.boolean();
|
|
20
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
21
|
+
expect(typeof value).toBe('boolean');
|
|
22
|
+
expect(schema.validate(value).success).toBe(true);
|
|
23
|
+
});
|
|
24
|
+
it('generates object', () => {
|
|
25
|
+
const schema = schema_1.Schema.object({ name: schema_1.Schema.string(), age: schema_1.Schema.number() });
|
|
26
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
27
|
+
expect(value).toHaveProperty('name');
|
|
28
|
+
expect(value).toHaveProperty('age');
|
|
29
|
+
expect(schema.validate(value).success).toBe(true);
|
|
30
|
+
});
|
|
31
|
+
it('generateMany returns array', () => {
|
|
32
|
+
const schema = schema_1.Schema.object({ id: schema_1.Schema.number() });
|
|
33
|
+
const values = schema_faker_1.SchemaFaker.generateMany(schema, 3);
|
|
34
|
+
expect(values).toHaveLength(3);
|
|
35
|
+
values.forEach((v) => expect(schema.validate(v).success).toBe(true));
|
|
36
|
+
});
|
|
37
|
+
it('generates literal', () => {
|
|
38
|
+
const schema = schema_1.Schema.literal('active');
|
|
39
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
40
|
+
expect(value).toBe('active');
|
|
41
|
+
});
|
|
42
|
+
it('generates array', () => {
|
|
43
|
+
const schema = schema_1.Schema.array(schema_1.Schema.number());
|
|
44
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
45
|
+
expect(Array.isArray(value)).toBe(true);
|
|
46
|
+
expect(schema.validate(value).success).toBe(true);
|
|
47
|
+
});
|
|
48
|
+
it('generates union', () => {
|
|
49
|
+
const schema = schema_1.Schema.union([schema_1.Schema.literal('a'), schema_1.Schema.literal('b')]);
|
|
50
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
51
|
+
expect(['a', 'b']).toContain(value);
|
|
52
|
+
});
|
|
53
|
+
it('constructor with custom array length options', () => {
|
|
54
|
+
const faker = new schema_faker_1.SchemaFaker({ arrayMinLength: 2, arrayMaxLength: 4 });
|
|
55
|
+
const schema = schema_1.Schema.array(schema_1.Schema.number());
|
|
56
|
+
const value = faker.generate(schema);
|
|
57
|
+
expect(Array.isArray(value)).toBe(true);
|
|
58
|
+
expect(value.length).toBeGreaterThanOrEqual(2);
|
|
59
|
+
expect(value.length).toBeLessThanOrEqual(4);
|
|
60
|
+
});
|
|
61
|
+
it('generateFromJsonSchema handles nullable type array', () => {
|
|
62
|
+
const schema = schema_1.Schema.string().nullable();
|
|
63
|
+
const value = schema_faker_1.SchemaFaker.generate(schema);
|
|
64
|
+
expect(value === null || typeof value === 'string').toBe(true);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -14,11 +14,17 @@ describe('built-in transformers', () => {
|
|
|
14
14
|
it('lowercases string', () => {
|
|
15
15
|
expect((0, built_in_transformers_1.toLowerCase)('HELLO')).toBe('hello');
|
|
16
16
|
});
|
|
17
|
+
it('converts non-string', () => {
|
|
18
|
+
expect((0, built_in_transformers_1.toLowerCase)(123)).toBe('123');
|
|
19
|
+
});
|
|
17
20
|
});
|
|
18
21
|
describe('toUpperCase', () => {
|
|
19
22
|
it('uppercases string', () => {
|
|
20
23
|
expect((0, built_in_transformers_1.toUpperCase)('hello')).toBe('HELLO');
|
|
21
24
|
});
|
|
25
|
+
it('converts non-string', () => {
|
|
26
|
+
expect((0, built_in_transformers_1.toUpperCase)(123)).toBe('123');
|
|
27
|
+
});
|
|
22
28
|
});
|
|
23
29
|
describe('parseJson', () => {
|
|
24
30
|
it('parses JSON string', () => {
|
|
@@ -41,17 +47,39 @@ describe('built-in transformers', () => {
|
|
|
41
47
|
it('handles missing keys', () => {
|
|
42
48
|
expect((0, built_in_transformers_1.pick)(['x'])({})).toEqual({});
|
|
43
49
|
});
|
|
50
|
+
it('returns empty for null', () => {
|
|
51
|
+
expect((0, built_in_transformers_1.pick)(['a'])(null)).toEqual({});
|
|
52
|
+
});
|
|
53
|
+
it('returns empty for non-object', () => {
|
|
54
|
+
expect((0, built_in_transformers_1.pick)(['a'])('string')).toEqual({});
|
|
55
|
+
});
|
|
44
56
|
});
|
|
45
57
|
describe('omit', () => {
|
|
46
58
|
it('omits specified keys', () => {
|
|
47
59
|
const fn = (0, built_in_transformers_1.omit)(['b']);
|
|
48
60
|
expect(fn({ a: 1, b: 2, c: 3 })).toEqual({ a: 1, c: 3 });
|
|
49
61
|
});
|
|
62
|
+
it('returns empty for null', () => {
|
|
63
|
+
expect((0, built_in_transformers_1.omit)(['a'])(null)).toEqual({});
|
|
64
|
+
});
|
|
65
|
+
it('returns empty for non-object', () => {
|
|
66
|
+
expect((0, built_in_transformers_1.omit)(['a'])(123)).toEqual({});
|
|
67
|
+
});
|
|
50
68
|
});
|
|
51
69
|
describe('renameKeys', () => {
|
|
52
70
|
it('renames keys', () => {
|
|
53
71
|
const fn = (0, built_in_transformers_1.renameKeys)({ oldName: 'newName' });
|
|
54
72
|
expect(fn({ oldName: 'value', keep: 1 })).toEqual({ newName: 'value', keep: 1 });
|
|
55
73
|
});
|
|
74
|
+
it('returns empty for null', () => {
|
|
75
|
+
expect((0, built_in_transformers_1.renameKeys)({ a: 'b' })(null)).toEqual({});
|
|
76
|
+
});
|
|
77
|
+
it('returns empty for non-object', () => {
|
|
78
|
+
expect((0, built_in_transformers_1.renameKeys)({ a: 'b' })(123)).toEqual({});
|
|
79
|
+
});
|
|
80
|
+
it('keeps keys not in mapping', () => {
|
|
81
|
+
const fn = (0, built_in_transformers_1.renameKeys)({ a: 'A' });
|
|
82
|
+
expect(fn({ a: 1, b: 2 })).toEqual({ A: 1, b: 2 });
|
|
83
|
+
});
|
|
56
84
|
});
|
|
57
85
|
});
|
|
@@ -29,4 +29,14 @@ describe('TransformerService', () => {
|
|
|
29
29
|
const result = await filter([1, 2, 3, 4]);
|
|
30
30
|
expect(result).toEqual([3, 4]);
|
|
31
31
|
});
|
|
32
|
+
it('apply returns sync result without awaiting', async () => {
|
|
33
|
+
service.register('sync', (x) => x + 1);
|
|
34
|
+
const result = await service.apply('sync', 10);
|
|
35
|
+
expect(result).toBe(11);
|
|
36
|
+
});
|
|
37
|
+
it('filter with async predicate', async () => {
|
|
38
|
+
const filter = service.filter(async (x) => x > 1);
|
|
39
|
+
const result = await filter([1, 2, 3]);
|
|
40
|
+
expect(result).toEqual([2, 3]);
|
|
41
|
+
});
|
|
32
42
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hazeljs/data",
|
|
3
|
-
"version": "0.2.0-beta.
|
|
3
|
+
"version": "0.2.0-beta.69",
|
|
4
4
|
"description": "Data Processing & ETL for HazelJS framework",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -49,5 +49,5 @@
|
|
|
49
49
|
"url": "https://github.com/hazeljs/hazel-js/issues"
|
|
50
50
|
},
|
|
51
51
|
"homepage": "https://hazeljs.com",
|
|
52
|
-
"gitHead": "
|
|
52
|
+
"gitHead": "a0aa75c1e4607c4e362f907163d248066d567517"
|
|
53
53
|
}
|