@hazeljs/data 0.2.0-beta.68 → 0.2.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -61
- package/dist/connectors/connector.interface.d.ts +29 -0
- package/dist/connectors/connector.interface.d.ts.map +1 -0
- package/dist/connectors/connector.interface.js +6 -0
- package/dist/connectors/csv.connector.d.ts +63 -0
- package/dist/connectors/csv.connector.d.ts.map +1 -0
- package/dist/connectors/csv.connector.js +147 -0
- package/dist/connectors/http.connector.d.ts +68 -0
- package/dist/connectors/http.connector.d.ts.map +1 -0
- package/dist/connectors/http.connector.js +131 -0
- package/dist/connectors/index.d.ts +7 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +12 -0
- package/dist/connectors/memory.connector.d.ts +38 -0
- package/dist/connectors/memory.connector.d.ts.map +1 -0
- package/dist/connectors/memory.connector.js +56 -0
- package/dist/connectors/memory.connector.test.d.ts +2 -0
- package/dist/connectors/memory.connector.test.d.ts.map +1 -0
- package/dist/connectors/memory.connector.test.js +43 -0
- package/dist/data.types.d.ts +16 -0
- package/dist/data.types.d.ts.map +1 -1
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.d.ts.map +1 -1
- package/dist/decorators/index.js +8 -1
- package/dist/decorators/pii.decorator.d.ts +59 -0
- package/dist/decorators/pii.decorator.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.js +197 -0
- package/dist/decorators/pii.decorator.test.d.ts +2 -0
- package/dist/decorators/pii.decorator.test.d.ts.map +1 -0
- package/dist/decorators/pii.decorator.test.js +150 -0
- package/dist/decorators/pipeline.decorator.js +1 -1
- package/dist/decorators/pipeline.decorator.test.js +8 -0
- package/dist/decorators/transform.decorator.d.ts +9 -1
- package/dist/decorators/transform.decorator.d.ts.map +1 -1
- package/dist/decorators/transform.decorator.js +4 -0
- package/dist/decorators/validate.decorator.d.ts +5 -1
- package/dist/decorators/validate.decorator.d.ts.map +1 -1
- package/dist/decorators/validate.decorator.js +4 -0
- package/dist/flink.service.d.ts +30 -0
- package/dist/flink.service.d.ts.map +1 -1
- package/dist/flink.service.js +50 -2
- package/dist/index.d.ts +13 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -8
- package/dist/pipelines/etl.service.d.ts +41 -2
- package/dist/pipelines/etl.service.d.ts.map +1 -1
- package/dist/pipelines/etl.service.js +143 -6
- package/dist/pipelines/etl.service.test.js +215 -0
- package/dist/pipelines/pipeline.builder.d.ts +86 -13
- package/dist/pipelines/pipeline.builder.d.ts.map +1 -1
- package/dist/pipelines/pipeline.builder.js +177 -27
- package/dist/pipelines/pipeline.builder.test.js +160 -12
- package/dist/pipelines/stream.service.test.js +49 -0
- package/dist/quality/quality.service.d.ts +67 -5
- package/dist/quality/quality.service.d.ts.map +1 -1
- package/dist/quality/quality.service.js +259 -20
- package/dist/quality/quality.service.test.js +94 -0
- package/dist/schema/schema.d.ts +92 -12
- package/dist/schema/schema.d.ts.map +1 -1
- package/dist/schema/schema.js +395 -83
- package/dist/schema/schema.test.js +292 -0
- package/dist/streaming/flink/flink.client.d.ts +41 -3
- package/dist/streaming/flink/flink.client.d.ts.map +1 -1
- package/dist/streaming/flink/flink.client.js +171 -8
- package/dist/streaming/flink/flink.client.test.js +2 -2
- package/dist/streaming/flink/flink.job.d.ts +2 -1
- package/dist/streaming/flink/flink.job.d.ts.map +1 -1
- package/dist/streaming/flink/flink.job.js +2 -2
- package/dist/streaming/stream.processor.d.ts +56 -2
- package/dist/streaming/stream.processor.d.ts.map +1 -1
- package/dist/streaming/stream.processor.js +149 -2
- package/dist/streaming/stream.processor.test.js +99 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts +2 -0
- package/dist/streaming/stream.processor.windowing.test.d.ts.map +1 -0
- package/dist/streaming/stream.processor.windowing.test.js +69 -0
- package/dist/telemetry/telemetry.d.ts +124 -0
- package/dist/telemetry/telemetry.d.ts.map +1 -0
- package/dist/telemetry/telemetry.js +259 -0
- package/dist/telemetry/telemetry.test.d.ts +2 -0
- package/dist/telemetry/telemetry.test.d.ts.map +1 -0
- package/dist/telemetry/telemetry.test.js +51 -0
- package/dist/testing/index.d.ts +12 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/pipeline-test-harness.d.ts +40 -0
- package/dist/testing/pipeline-test-harness.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.js +55 -0
- package/dist/testing/pipeline-test-harness.test.d.ts +2 -0
- package/dist/testing/pipeline-test-harness.test.d.ts.map +1 -0
- package/dist/testing/pipeline-test-harness.test.js +102 -0
- package/dist/testing/schema-faker.d.ts +32 -0
- package/dist/testing/schema-faker.d.ts.map +1 -0
- package/dist/testing/schema-faker.js +91 -0
- package/dist/testing/schema-faker.test.d.ts +2 -0
- package/dist/testing/schema-faker.test.d.ts.map +1 -0
- package/dist/testing/schema-faker.test.js +66 -0
- package/dist/transformers/built-in.transformers.test.js +28 -0
- package/dist/transformers/transformer.service.test.js +10 -0
- package/package.json +2 -2
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HttpSink = exports.HttpSource = void 0;
|
|
4
|
+
function getNestedValue(obj, path) {
|
|
5
|
+
return path.split('.').reduce((cur, key) => {
|
|
6
|
+
if (cur === null || cur === undefined || typeof cur !== 'object')
|
|
7
|
+
return undefined;
|
|
8
|
+
return cur[key];
|
|
9
|
+
}, obj);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* HTTP API data source — reads records from a REST API.
|
|
13
|
+
* Supports pagination via `nextPagePath`.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* const source = new HttpSource({
|
|
17
|
+
* url: 'https://api.example.com/users',
|
|
18
|
+
* dataPath: 'data',
|
|
19
|
+
* nextPagePath: 'meta.next',
|
|
20
|
+
* });
|
|
21
|
+
*/
|
|
22
|
+
class HttpSource {
|
|
23
|
+
constructor(options) {
|
|
24
|
+
this.name = options.name ?? `http:${options.url}`;
|
|
25
|
+
this.options = options;
|
|
26
|
+
}
|
|
27
|
+
async open() {
|
|
28
|
+
// Validate connectivity on open
|
|
29
|
+
const res = await this.fetchPage(this.options.url);
|
|
30
|
+
if (!res.ok)
|
|
31
|
+
throw new Error(`HttpSource: Cannot reach ${this.options.url} (${res.status})`);
|
|
32
|
+
}
|
|
33
|
+
async close() { }
|
|
34
|
+
async readAll() {
|
|
35
|
+
const all = [];
|
|
36
|
+
for await (const r of this.read())
|
|
37
|
+
all.push(r);
|
|
38
|
+
return all;
|
|
39
|
+
}
|
|
40
|
+
async *read() {
|
|
41
|
+
let url = this.options.url;
|
|
42
|
+
while (url) {
|
|
43
|
+
const res = await this.fetchPage(url);
|
|
44
|
+
if (!res.ok)
|
|
45
|
+
throw new Error(`HttpSource: ${res.status} ${await res.text()}`);
|
|
46
|
+
const body = await res.json();
|
|
47
|
+
const items = this.options.dataPath ? getNestedValue(body, this.options.dataPath) : body;
|
|
48
|
+
if (Array.isArray(items)) {
|
|
49
|
+
for (const item of items)
|
|
50
|
+
yield item;
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
yield body;
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
url = this.options.nextPagePath
|
|
57
|
+
? (getNestedValue(body, this.options.nextPagePath) ?? null)
|
|
58
|
+
: null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
fetchPage(url) {
|
|
62
|
+
const controller = new AbortController();
|
|
63
|
+
if (this.options.timeoutMs) {
|
|
64
|
+
setTimeout(() => controller.abort(), this.options.timeoutMs);
|
|
65
|
+
}
|
|
66
|
+
return fetch(url, {
|
|
67
|
+
method: this.options.method ?? 'GET',
|
|
68
|
+
headers: { 'Content-Type': 'application/json', ...this.options.headers },
|
|
69
|
+
body: this.options.body ? JSON.stringify(this.options.body) : undefined,
|
|
70
|
+
signal: controller.signal,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.HttpSource = HttpSource;
|
|
75
|
+
/**
|
|
76
|
+
* HTTP API data sink — writes records to a REST API endpoint.
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* const sink = new HttpSink({
|
|
80
|
+
* url: 'https://api.example.com/ingest',
|
|
81
|
+
* method: 'POST',
|
|
82
|
+
* batchSize: 100,
|
|
83
|
+
* bodyKey: 'records',
|
|
84
|
+
* });
|
|
85
|
+
*/
|
|
86
|
+
class HttpSink {
|
|
87
|
+
constructor(options) {
|
|
88
|
+
this.buffer = [];
|
|
89
|
+
this.name = options.name ?? `http:${options.url}`;
|
|
90
|
+
this.options = { batchSize: 1, ...options };
|
|
91
|
+
}
|
|
92
|
+
async open() { }
|
|
93
|
+
async close() {
|
|
94
|
+
if (this.buffer.length > 0)
|
|
95
|
+
await this.flush();
|
|
96
|
+
}
|
|
97
|
+
async write(record) {
|
|
98
|
+
this.buffer.push(record);
|
|
99
|
+
if (this.buffer.length >= (this.options.batchSize ?? 1)) {
|
|
100
|
+
await this.flush();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
async writeBatch(records) {
|
|
104
|
+
this.buffer.push(...records);
|
|
105
|
+
await this.flush();
|
|
106
|
+
}
|
|
107
|
+
async flush() {
|
|
108
|
+
if (this.buffer.length === 0)
|
|
109
|
+
return;
|
|
110
|
+
const payload = this.options.bodyKey
|
|
111
|
+
? { [this.options.bodyKey]: this.buffer }
|
|
112
|
+
: this.buffer.length === 1
|
|
113
|
+
? this.buffer[0]
|
|
114
|
+
: this.buffer;
|
|
115
|
+
const controller = new AbortController();
|
|
116
|
+
if (this.options.timeoutMs) {
|
|
117
|
+
setTimeout(() => controller.abort(), this.options.timeoutMs);
|
|
118
|
+
}
|
|
119
|
+
const res = await fetch(this.options.url, {
|
|
120
|
+
method: this.options.method ?? 'POST',
|
|
121
|
+
headers: { 'Content-Type': 'application/json', ...this.options.headers },
|
|
122
|
+
body: JSON.stringify(payload),
|
|
123
|
+
signal: controller.signal,
|
|
124
|
+
});
|
|
125
|
+
if (!res.ok) {
|
|
126
|
+
throw new Error(`HttpSink: ${res.status} ${await res.text()}`);
|
|
127
|
+
}
|
|
128
|
+
this.buffer = [];
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
exports.HttpSink = HttpSink;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export type { DataSource, DataSink, ConnectorOptions } from './connector.interface';
|
|
2
|
+
export { CsvSource, CsvSink } from './csv.connector';
|
|
3
|
+
export type { CsvSourceOptions, CsvSinkOptions } from './csv.connector';
|
|
4
|
+
export { HttpSource, HttpSink } from './http.connector';
|
|
5
|
+
export type { HttpSourceOptions, HttpSinkOptions } from './http.connector';
|
|
6
|
+
export { MemorySource, MemorySink } from './memory.connector';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/connectors/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AACrD,YAAY,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACxD,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MemorySink = exports.MemorySource = exports.HttpSink = exports.HttpSource = exports.CsvSink = exports.CsvSource = void 0;
|
|
4
|
+
var csv_connector_1 = require("./csv.connector");
|
|
5
|
+
Object.defineProperty(exports, "CsvSource", { enumerable: true, get: function () { return csv_connector_1.CsvSource; } });
|
|
6
|
+
Object.defineProperty(exports, "CsvSink", { enumerable: true, get: function () { return csv_connector_1.CsvSink; } });
|
|
7
|
+
var http_connector_1 = require("./http.connector");
|
|
8
|
+
Object.defineProperty(exports, "HttpSource", { enumerable: true, get: function () { return http_connector_1.HttpSource; } });
|
|
9
|
+
Object.defineProperty(exports, "HttpSink", { enumerable: true, get: function () { return http_connector_1.HttpSink; } });
|
|
10
|
+
var memory_connector_1 = require("./memory.connector");
|
|
11
|
+
Object.defineProperty(exports, "MemorySource", { enumerable: true, get: function () { return memory_connector_1.MemorySource; } });
|
|
12
|
+
Object.defineProperty(exports, "MemorySink", { enumerable: true, get: function () { return memory_connector_1.MemorySink; } });
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { DataSource, DataSink } from './connector.interface';
|
|
2
|
+
/**
|
|
3
|
+
* In-memory data source — wraps an array for use in pipelines and tests.
|
|
4
|
+
*
|
|
5
|
+
* @example
|
|
6
|
+
* const source = new MemorySource([{ id: 1 }, { id: 2 }]);
|
|
7
|
+
* const records = await source.readAll();
|
|
8
|
+
*/
|
|
9
|
+
export declare class MemorySource<T = unknown> implements DataSource<T> {
|
|
10
|
+
readonly name: string;
|
|
11
|
+
private readonly records;
|
|
12
|
+
constructor(records: T[], name?: string);
|
|
13
|
+
open(): Promise<void>;
|
|
14
|
+
close(): Promise<void>;
|
|
15
|
+
readAll(): Promise<T[]>;
|
|
16
|
+
read(): AsyncGenerator<T>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* In-memory data sink — captures written records for inspection.
|
|
20
|
+
* Useful in tests and pipeline prototyping.
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* const sink = new MemorySink<User>();
|
|
24
|
+
* await pipeline.run(source, sink);
|
|
25
|
+
* console.log(sink.records);
|
|
26
|
+
*/
|
|
27
|
+
export declare class MemorySink<T = unknown> implements DataSink<T> {
|
|
28
|
+
readonly name: string;
|
|
29
|
+
private _records;
|
|
30
|
+
constructor(name?: string);
|
|
31
|
+
open(): Promise<void>;
|
|
32
|
+
close(): Promise<void>;
|
|
33
|
+
write(record: T): Promise<void>;
|
|
34
|
+
writeBatch(records: T[]): Promise<void>;
|
|
35
|
+
get records(): readonly T[];
|
|
36
|
+
clear(): void;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=memory.connector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.connector.d.ts","sourceRoot":"","sources":["../../src/connectors/memory.connector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAElE;;;;;;GAMG;AACH,qBAAa,YAAY,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,UAAU,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAM;gBAElB,OAAO,EAAE,CAAC,EAAE,EAAE,IAAI,SAAkB;IAK1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IACrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAEtB,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;IAItB,IAAI,IAAI,cAAc,CAAC,CAAC,CAAC;CAGjC;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,QAAQ,CAAC,CAAC,CAAC;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAW;gBAEf,IAAI,SAAgB;IAI1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IACrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAEtB,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/B,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,IAAI,OAAO,IAAI,SAAS,CAAC,EAAE,CAE1B;IAED,KAAK,IAAI,IAAI;CAGd"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MemorySink = exports.MemorySource = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* In-memory data source — wraps an array for use in pipelines and tests.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* const source = new MemorySource([{ id: 1 }, { id: 2 }]);
|
|
9
|
+
* const records = await source.readAll();
|
|
10
|
+
*/
|
|
11
|
+
class MemorySource {
|
|
12
|
+
constructor(records, name = 'memory:source') {
|
|
13
|
+
this.name = name;
|
|
14
|
+
this.records = [...records];
|
|
15
|
+
}
|
|
16
|
+
async open() { }
|
|
17
|
+
async close() { }
|
|
18
|
+
async readAll() {
|
|
19
|
+
return [...this.records];
|
|
20
|
+
}
|
|
21
|
+
async *read() {
|
|
22
|
+
for (const r of this.records)
|
|
23
|
+
yield r;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
exports.MemorySource = MemorySource;
|
|
27
|
+
/**
|
|
28
|
+
* In-memory data sink — captures written records for inspection.
|
|
29
|
+
* Useful in tests and pipeline prototyping.
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* const sink = new MemorySink<User>();
|
|
33
|
+
* await pipeline.run(source, sink);
|
|
34
|
+
* console.log(sink.records);
|
|
35
|
+
*/
|
|
36
|
+
class MemorySink {
|
|
37
|
+
constructor(name = 'memory:sink') {
|
|
38
|
+
this._records = [];
|
|
39
|
+
this.name = name;
|
|
40
|
+
}
|
|
41
|
+
async open() { }
|
|
42
|
+
async close() { }
|
|
43
|
+
async write(record) {
|
|
44
|
+
this._records.push(record);
|
|
45
|
+
}
|
|
46
|
+
async writeBatch(records) {
|
|
47
|
+
this._records.push(...records);
|
|
48
|
+
}
|
|
49
|
+
get records() {
|
|
50
|
+
return this._records;
|
|
51
|
+
}
|
|
52
|
+
clear() {
|
|
53
|
+
this._records = [];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
exports.MemorySink = MemorySink;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.connector.test.d.ts","sourceRoot":"","sources":["../../src/connectors/memory.connector.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const memory_connector_1 = require("./memory.connector");
|
|
4
|
+
describe('MemorySource', () => {
|
|
5
|
+
it('readAll returns all records', async () => {
|
|
6
|
+
const source = new memory_connector_1.MemorySource([{ id: 1 }, { id: 2 }]);
|
|
7
|
+
await source.open();
|
|
8
|
+
const records = await source.readAll();
|
|
9
|
+
expect(records).toHaveLength(2);
|
|
10
|
+
expect(records[0]).toEqual({ id: 1 });
|
|
11
|
+
await source.close();
|
|
12
|
+
});
|
|
13
|
+
it('read yields records', async () => {
|
|
14
|
+
const source = new memory_connector_1.MemorySource([1, 2, 3]);
|
|
15
|
+
const out = [];
|
|
16
|
+
for await (const r of source.read())
|
|
17
|
+
out.push(r);
|
|
18
|
+
expect(out).toEqual([1, 2, 3]);
|
|
19
|
+
});
|
|
20
|
+
});
|
|
21
|
+
describe('MemorySink', () => {
|
|
22
|
+
it('captures written records', async () => {
|
|
23
|
+
const sink = new memory_connector_1.MemorySink();
|
|
24
|
+
await sink.open();
|
|
25
|
+
await sink.write({ x: 1 });
|
|
26
|
+
await sink.write({ x: 2 });
|
|
27
|
+
await sink.close();
|
|
28
|
+
expect(sink.records).toHaveLength(2);
|
|
29
|
+
expect(sink.records[0]).toEqual({ x: 1 });
|
|
30
|
+
});
|
|
31
|
+
it('writeBatch captures batch', async () => {
|
|
32
|
+
const sink = new memory_connector_1.MemorySink();
|
|
33
|
+
await sink.open();
|
|
34
|
+
await sink.writeBatch([1, 2, 3]);
|
|
35
|
+
expect(sink.records).toEqual([1, 2, 3]);
|
|
36
|
+
});
|
|
37
|
+
it('clear resets records', async () => {
|
|
38
|
+
const sink = new memory_connector_1.MemorySink();
|
|
39
|
+
await sink.write(1);
|
|
40
|
+
sink.clear();
|
|
41
|
+
expect(sink.records).toHaveLength(0);
|
|
42
|
+
});
|
|
43
|
+
});
|
package/dist/data.types.d.ts
CHANGED
|
@@ -1,11 +1,27 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @hazeljs/data - Type definitions for ETL and streaming
|
|
3
3
|
*/
|
|
4
|
+
export interface RetryConfig {
|
|
5
|
+
attempts: number;
|
|
6
|
+
delay?: number;
|
|
7
|
+
backoff?: 'fixed' | 'exponential';
|
|
8
|
+
}
|
|
9
|
+
export interface DLQConfig {
|
|
10
|
+
handler: (item: unknown, error: Error, stepName: string) => void | Promise<void>;
|
|
11
|
+
}
|
|
4
12
|
export interface PipelineStepMetadata {
|
|
5
13
|
step: number;
|
|
6
14
|
name: string;
|
|
7
15
|
type: 'transform' | 'validate';
|
|
8
16
|
schema?: unknown;
|
|
17
|
+
/** Run step only when predicate returns true */
|
|
18
|
+
when?: (data: unknown) => boolean;
|
|
19
|
+
/** Retry configuration for this step */
|
|
20
|
+
retry?: RetryConfig;
|
|
21
|
+
/** Per-step timeout in milliseconds */
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
/** Dead letter queue handler - called instead of throwing on failure */
|
|
24
|
+
dlq?: DLQConfig;
|
|
9
25
|
}
|
|
10
26
|
export interface StreamMetadata {
|
|
11
27
|
name: string;
|
package/dist/data.types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.types.d.ts","sourceRoot":"","sources":["../src/data.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"data.types.d.ts","sourceRoot":"","sources":["../src/data.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,GAAG,aAAa,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClF;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IAClC,wCAAwC;IACxC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,uCAAuC;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE;QAChB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,gBAAgB,CAAC,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,OAAO,CAAC,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -2,4 +2,5 @@ export { Pipeline, getPipelineMetadata, hasPipelineMetadata, type PipelineOption
|
|
|
2
2
|
export { Transform, getTransformMetadata, type TransformOptions } from './transform.decorator';
|
|
3
3
|
export { Validate, getValidateMetadata, type ValidateOptions } from './validate.decorator';
|
|
4
4
|
export { Stream, getStreamMetadata, hasStreamMetadata, type StreamOptions, } from './stream.decorator';
|
|
5
|
+
export { Mask, Redact, Encrypt, Decrypt, getMaskMetadata, getRedactMetadata, type MaskOptions, type RedactOptions, type EncryptOptions, type DecryptOptions, } from './pii.decorator';
|
|
5
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/decorators/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,mBAAmB,EACnB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,EACL,MAAM,EACN,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,aAAa,GACnB,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/decorators/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,mBAAmB,EACnB,mBAAmB,EACnB,KAAK,eAAe,GACrB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,EACL,MAAM,EACN,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,aAAa,GACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,IAAI,EACJ,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,EACf,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,GACpB,MAAM,iBAAiB,CAAC"}
|
package/dist/decorators/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.hasStreamMetadata = exports.getStreamMetadata = exports.Stream = exports.getValidateMetadata = exports.Validate = exports.getTransformMetadata = exports.Transform = exports.hasPipelineMetadata = exports.getPipelineMetadata = exports.Pipeline = void 0;
|
|
3
|
+
exports.getRedactMetadata = exports.getMaskMetadata = exports.Decrypt = exports.Encrypt = exports.Redact = exports.Mask = exports.hasStreamMetadata = exports.getStreamMetadata = exports.Stream = exports.getValidateMetadata = exports.Validate = exports.getTransformMetadata = exports.Transform = exports.hasPipelineMetadata = exports.getPipelineMetadata = exports.Pipeline = void 0;
|
|
4
4
|
var pipeline_decorator_1 = require("./pipeline.decorator");
|
|
5
5
|
Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return pipeline_decorator_1.Pipeline; } });
|
|
6
6
|
Object.defineProperty(exports, "getPipelineMetadata", { enumerable: true, get: function () { return pipeline_decorator_1.getPipelineMetadata; } });
|
|
@@ -15,3 +15,10 @@ var stream_decorator_1 = require("./stream.decorator");
|
|
|
15
15
|
Object.defineProperty(exports, "Stream", { enumerable: true, get: function () { return stream_decorator_1.Stream; } });
|
|
16
16
|
Object.defineProperty(exports, "getStreamMetadata", { enumerable: true, get: function () { return stream_decorator_1.getStreamMetadata; } });
|
|
17
17
|
Object.defineProperty(exports, "hasStreamMetadata", { enumerable: true, get: function () { return stream_decorator_1.hasStreamMetadata; } });
|
|
18
|
+
var pii_decorator_1 = require("./pii.decorator");
|
|
19
|
+
Object.defineProperty(exports, "Mask", { enumerable: true, get: function () { return pii_decorator_1.Mask; } });
|
|
20
|
+
Object.defineProperty(exports, "Redact", { enumerable: true, get: function () { return pii_decorator_1.Redact; } });
|
|
21
|
+
Object.defineProperty(exports, "Encrypt", { enumerable: true, get: function () { return pii_decorator_1.Encrypt; } });
|
|
22
|
+
Object.defineProperty(exports, "Decrypt", { enumerable: true, get: function () { return pii_decorator_1.Decrypt; } });
|
|
23
|
+
Object.defineProperty(exports, "getMaskMetadata", { enumerable: true, get: function () { return pii_decorator_1.getMaskMetadata; } });
|
|
24
|
+
Object.defineProperty(exports, "getRedactMetadata", { enumerable: true, get: function () { return pii_decorator_1.getRedactMetadata; } });
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import 'reflect-metadata';
|
|
2
|
+
export interface MaskOptions {
|
|
3
|
+
/** Fields to mask */
|
|
4
|
+
fields: string[];
|
|
5
|
+
/** Replacement string (default: "****") */
|
|
6
|
+
replacement?: string;
|
|
7
|
+
/** Show last N characters of the value (default: 0) */
|
|
8
|
+
showLast?: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Mask specified fields before the method runs.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* @Transform({ step: 1, name: 'sanitize' })
|
|
15
|
+
* @Mask({ fields: ['email', 'ssn'] })
|
|
16
|
+
* sanitize(data: User) { return data; }
|
|
17
|
+
*/
|
|
18
|
+
export declare function Mask(fieldsOrOptions: string[] | MaskOptions): MethodDecorator;
|
|
19
|
+
export declare function getMaskMetadata(target: object, key: string | symbol): MaskOptions | undefined;
|
|
20
|
+
export interface RedactOptions {
|
|
21
|
+
fields: string[];
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Remove specified fields entirely before the method runs.
|
|
25
|
+
*
|
|
26
|
+
* @example
|
|
27
|
+
* @Transform({ step: 2, name: 'redact' })
|
|
28
|
+
* @Redact({ fields: ['password', 'secretToken'] })
|
|
29
|
+
* redactSecrets(data: User) { return data; }
|
|
30
|
+
*/
|
|
31
|
+
export declare function Redact(fieldsOrOptions: string[] | RedactOptions): MethodDecorator;
|
|
32
|
+
export declare function getRedactMetadata(target: object, key: string | symbol): RedactOptions | undefined;
|
|
33
|
+
export interface EncryptOptions {
|
|
34
|
+
fields: string[];
|
|
35
|
+
/** 32-byte AES-256-GCM key (hex string or Buffer) */
|
|
36
|
+
key: string | Buffer;
|
|
37
|
+
/** Optional AAD (additional authenticated data) for GCM */
|
|
38
|
+
aad?: string;
|
|
39
|
+
}
|
|
40
|
+
export interface DecryptOptions {
|
|
41
|
+
fields: string[];
|
|
42
|
+
key: string | Buffer;
|
|
43
|
+
aad?: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* AES-256-GCM encrypt specified fields before the method runs.
|
|
47
|
+
* Encrypted values are stored as "iv:authTag:ciphertext" (all hex-encoded).
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* @Transform({ step: 3, name: 'encrypt-pii' })
|
|
51
|
+
* @Encrypt({ fields: ['ssn', 'dob'], key: process.env.FIELD_ENCRYPTION_KEY! })
|
|
52
|
+
* encryptPii(data: User) { return data; }
|
|
53
|
+
*/
|
|
54
|
+
export declare function Encrypt(options: EncryptOptions): MethodDecorator;
|
|
55
|
+
/**
|
|
56
|
+
* Decrypt fields that were encrypted with @Encrypt.
|
|
57
|
+
*/
|
|
58
|
+
export declare function Decrypt(options: DecryptOptions): MethodDecorator;
|
|
59
|
+
//# sourceMappingURL=pii.decorator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pii.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/pii.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAoB1B,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,GAAG,WAAW,GAAG,eAAe,CAc7E;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,GAAG,SAAS,CAE7F;AAqCD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,wBAAgB,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,GAAG,aAAa,GAAG,eAAe,CAcjF;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAEjG;AAaD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,qDAAqD;IACrD,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;GAQG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,eAAe,CAUhE;AA4BD;;GAEG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,eAAe,CAQhE"}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Mask = Mask;
|
|
4
|
+
exports.getMaskMetadata = getMaskMetadata;
|
|
5
|
+
exports.Redact = Redact;
|
|
6
|
+
exports.getRedactMetadata = getRedactMetadata;
|
|
7
|
+
exports.Encrypt = Encrypt;
|
|
8
|
+
exports.Decrypt = Decrypt;
|
|
9
|
+
require("reflect-metadata");
|
|
10
|
+
/**
|
|
11
|
+
* PII-safety decorators for pipeline methods.
|
|
12
|
+
*
|
|
13
|
+
* These decorators run **before** the decorated method executes,
|
|
14
|
+
* modifying the data according to the specified operation.
|
|
15
|
+
*
|
|
16
|
+
* @Mask — replaces field values with "****"
|
|
17
|
+
* @Redact — removes fields entirely
|
|
18
|
+
* @Encrypt — AES-256-GCM encrypts field values (Node.js crypto required)
|
|
19
|
+
* @Decrypt — decrypts previously encrypted values
|
|
20
|
+
*/
|
|
21
|
+
const MASK_METADATA_KEY = 'hazel:data:pii:mask';
|
|
22
|
+
const REDACT_METADATA_KEY = 'hazel:data:pii:redact';
|
|
23
|
+
const ENCRYPT_METADATA_KEY = 'hazel:data:pii:encrypt';
|
|
24
|
+
/**
|
|
25
|
+
* Mask specified fields before the method runs.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* @Transform({ step: 1, name: 'sanitize' })
|
|
29
|
+
* @Mask({ fields: ['email', 'ssn'] })
|
|
30
|
+
* sanitize(data: User) { return data; }
|
|
31
|
+
*/
|
|
32
|
+
function Mask(fieldsOrOptions) {
|
|
33
|
+
const options = Array.isArray(fieldsOrOptions)
|
|
34
|
+
? { fields: fieldsOrOptions }
|
|
35
|
+
: fieldsOrOptions;
|
|
36
|
+
return (target, propertyKey, descriptor) => {
|
|
37
|
+
Reflect.defineMetadata(MASK_METADATA_KEY, options, target, propertyKey);
|
|
38
|
+
const original = descriptor.value;
|
|
39
|
+
descriptor.value = function (data, ...rest) {
|
|
40
|
+
return original.call(this, applyMask(data, options), ...rest);
|
|
41
|
+
};
|
|
42
|
+
return descriptor;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function getMaskMetadata(target, key) {
|
|
46
|
+
return Reflect.getMetadata(MASK_METADATA_KEY, target, key);
|
|
47
|
+
}
|
|
48
|
+
function applyMask(data, options) {
|
|
49
|
+
const { fields, replacement = '****', showLast = 0 } = options;
|
|
50
|
+
if (!data || typeof data !== 'object')
|
|
51
|
+
return data;
|
|
52
|
+
const result = { ...data };
|
|
53
|
+
for (const field of fields) {
|
|
54
|
+
const parts = field.split('.');
|
|
55
|
+
maskNested(result, parts, replacement, showLast);
|
|
56
|
+
}
|
|
57
|
+
return result;
|
|
58
|
+
}
|
|
59
|
+
function maskNested(obj, path, replacement, showLast) {
|
|
60
|
+
if (path.length === 1) {
|
|
61
|
+
const val = obj[path[0]];
|
|
62
|
+
if (typeof val === 'string' && showLast > 0) {
|
|
63
|
+
obj[path[0]] = replacement + val.slice(-showLast);
|
|
64
|
+
}
|
|
65
|
+
else if (val !== undefined && val !== null) {
|
|
66
|
+
obj[path[0]] = replacement;
|
|
67
|
+
}
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const nested = obj[path[0]];
|
|
71
|
+
if (nested && typeof nested === 'object' && !Array.isArray(nested)) {
|
|
72
|
+
maskNested(nested, path.slice(1), replacement, showLast);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Remove specified fields entirely before the method runs.
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* @Transform({ step: 2, name: 'redact' })
|
|
80
|
+
* @Redact({ fields: ['password', 'secretToken'] })
|
|
81
|
+
* redactSecrets(data: User) { return data; }
|
|
82
|
+
*/
|
|
83
|
+
function Redact(fieldsOrOptions) {
|
|
84
|
+
const options = Array.isArray(fieldsOrOptions)
|
|
85
|
+
? { fields: fieldsOrOptions }
|
|
86
|
+
: fieldsOrOptions;
|
|
87
|
+
return (target, propertyKey, descriptor) => {
|
|
88
|
+
Reflect.defineMetadata(REDACT_METADATA_KEY, options, target, propertyKey);
|
|
89
|
+
const original = descriptor.value;
|
|
90
|
+
descriptor.value = function (data, ...rest) {
|
|
91
|
+
return original.call(this, applyRedact(data, options), ...rest);
|
|
92
|
+
};
|
|
93
|
+
return descriptor;
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function getRedactMetadata(target, key) {
|
|
97
|
+
return Reflect.getMetadata(REDACT_METADATA_KEY, target, key);
|
|
98
|
+
}
|
|
99
|
+
function applyRedact(data, options) {
|
|
100
|
+
if (!data || typeof data !== 'object')
|
|
101
|
+
return data;
|
|
102
|
+
const result = { ...data };
|
|
103
|
+
for (const field of options.fields) {
|
|
104
|
+
delete result[field];
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* AES-256-GCM encrypt specified fields before the method runs.
|
|
110
|
+
* Encrypted values are stored as "iv:authTag:ciphertext" (all hex-encoded).
|
|
111
|
+
*
|
|
112
|
+
* @example
|
|
113
|
+
* @Transform({ step: 3, name: 'encrypt-pii' })
|
|
114
|
+
* @Encrypt({ fields: ['ssn', 'dob'], key: process.env.FIELD_ENCRYPTION_KEY! })
|
|
115
|
+
* encryptPii(data: User) { return data; }
|
|
116
|
+
*/
|
|
117
|
+
function Encrypt(options) {
|
|
118
|
+
return (target, propertyKey, descriptor) => {
|
|
119
|
+
Reflect.defineMetadata(ENCRYPT_METADATA_KEY, options, target, propertyKey);
|
|
120
|
+
const original = descriptor.value;
|
|
121
|
+
descriptor.value = function (data, ...rest) {
|
|
122
|
+
return original.call(this, applyEncrypt(data, options), ...rest);
|
|
123
|
+
};
|
|
124
|
+
return descriptor;
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function applyEncrypt(data, options) {
|
|
128
|
+
if (!data || typeof data !== 'object')
|
|
129
|
+
return data;
|
|
130
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
131
|
+
const crypto = require('crypto');
|
|
132
|
+
const keyBuf = typeof options.key === 'string' ? Buffer.from(options.key, 'hex') : options.key;
|
|
133
|
+
const result = { ...data };
|
|
134
|
+
for (const field of options.fields) {
|
|
135
|
+
const val = result[field];
|
|
136
|
+
if (val === undefined || val === null)
|
|
137
|
+
continue;
|
|
138
|
+
const iv = crypto.randomBytes(12);
|
|
139
|
+
const cipher = crypto.createCipheriv('aes-256-gcm', keyBuf, iv);
|
|
140
|
+
if (options.aad)
|
|
141
|
+
cipher.setAAD(Buffer.from(options.aad));
|
|
142
|
+
const plaintext = typeof val === 'string' ? val : JSON.stringify(val);
|
|
143
|
+
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
|
|
144
|
+
const authTag = cipher.getAuthTag();
|
|
145
|
+
result[field] = `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`;
|
|
146
|
+
}
|
|
147
|
+
return result;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Decrypt fields that were encrypted with @Encrypt.
|
|
151
|
+
*/
|
|
152
|
+
function Decrypt(options) {
|
|
153
|
+
return (_target, _propertyKey, descriptor) => {
|
|
154
|
+
const original = descriptor.value;
|
|
155
|
+
descriptor.value = function (data, ...rest) {
|
|
156
|
+
return original.call(this, applyDecrypt(data, options), ...rest);
|
|
157
|
+
};
|
|
158
|
+
return descriptor;
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function applyDecrypt(data, options) {
|
|
162
|
+
if (!data || typeof data !== 'object')
|
|
163
|
+
return data;
|
|
164
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
165
|
+
const crypto = require('crypto');
|
|
166
|
+
const keyBuf = typeof options.key === 'string' ? Buffer.from(options.key, 'hex') : options.key;
|
|
167
|
+
const result = { ...data };
|
|
168
|
+
for (const field of options.fields) {
|
|
169
|
+
const val = result[field];
|
|
170
|
+
if (typeof val !== 'string')
|
|
171
|
+
continue;
|
|
172
|
+
const parts = val.split(':');
|
|
173
|
+
if (parts.length !== 3)
|
|
174
|
+
continue;
|
|
175
|
+
try {
|
|
176
|
+
const [ivHex, authTagHex, ciphertextHex] = parts;
|
|
177
|
+
const iv = Buffer.from(ivHex, 'hex');
|
|
178
|
+
const authTag = Buffer.from(authTagHex, 'hex');
|
|
179
|
+
const ciphertext = Buffer.from(ciphertextHex, 'hex');
|
|
180
|
+
const decipher = crypto.createDecipheriv('aes-256-gcm', keyBuf, iv);
|
|
181
|
+
decipher.setAuthTag(authTag);
|
|
182
|
+
if (options.aad)
|
|
183
|
+
decipher.setAAD(Buffer.from(options.aad));
|
|
184
|
+
const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString('utf8');
|
|
185
|
+
try {
|
|
186
|
+
result[field] = JSON.parse(decrypted);
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
result[field] = decrypted;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// Leave as-is if decryption fails
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return result;
|
|
197
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pii.decorator.test.d.ts","sourceRoot":"","sources":["../../src/decorators/pii.decorator.test.ts"],"names":[],"mappings":""}
|