@hazeljs/data 0.2.0-beta.68 → 0.2.0-beta.69

package/dist/decorators/pii.decorator.test.js

@@ -0,0 +1,150 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const pipeline_decorator_1 = require("./pipeline.decorator");
+const transform_decorator_1 = require("./transform.decorator");
+const pii_decorator_1 = require("./pii.decorator");
+const etl_service_1 = require("../pipelines/etl.service");
+const schema_validator_1 = require("../validators/schema.validator");
+let MaskPipeline = class MaskPipeline {
+    mask(data) {
+        return data;
+    }
+};
+__decorate([
+    (0, transform_decorator_1.Transform)({ step: 1, name: 'mask' }),
+    (0, pii_decorator_1.Mask)({ fields: ['email', 'ssn'] }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", void 0)
+], MaskPipeline.prototype, "mask", null);
+MaskPipeline = __decorate([
+    (0, pipeline_decorator_1.Pipeline)('mask-pipeline')
+], MaskPipeline);
+let MaskShowLastPipeline = class MaskShowLastPipeline {
+    mask(data) {
+        return data;
+    }
+};
+__decorate([
+    (0, transform_decorator_1.Transform)({ step: 1, name: 'mask' }),
+    (0, pii_decorator_1.Mask)({ fields: ['phone'], replacement: '***', showLast: 4 }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", void 0)
+], MaskShowLastPipeline.prototype, "mask", null);
+MaskShowLastPipeline = __decorate([
+    (0, pipeline_decorator_1.Pipeline)('mask-showlast')
+], MaskShowLastPipeline);
+let RedactPipeline = class RedactPipeline {
+    redact(data) {
+        return data;
+    }
+};
+__decorate([
+    (0, transform_decorator_1.Transform)({ step: 1, name: 'redact' }),
+    (0, pii_decorator_1.Redact)({ fields: ['password', 'secret'] }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", void 0)
+], RedactPipeline.prototype, "redact", null);
+RedactPipeline = __decorate([
+    (0, pipeline_decorator_1.Pipeline)('redact-pipeline')
+], RedactPipeline);
+describe('@Mask decorator', () => {
+    it('masks specified fields', async () => {
+        const etl = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
+        const pipeline = new MaskPipeline();
+        const result = await etl.execute(pipeline, {
+            email: 'user@example.com',
+            ssn: '123-45-6789',
+            name: 'John',
+        });
+        expect(result.email).toBe('****');
+        expect(result.ssn).toBe('****');
+        expect(result.name).toBe('John');
+    });
+    it('showLast shows last N chars', async () => {
+        const etl = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
+        const pipeline = new MaskShowLastPipeline();
+        const result = await etl.execute(pipeline, {
+            phone: '1234567890',
+        });
+        expect(result.phone).toBe('***7890');
+    });
+    it('getMaskMetadata returns options', () => {
+        const meta = (0, pii_decorator_1.getMaskMetadata)(MaskPipeline.prototype, 'mask');
+        expect(meta?.fields).toContain('email');
+        expect(meta?.fields).toContain('ssn');
+    });
+});
+describe('@Redact decorator', () => {
+    it('removes specified fields', async () => {
+        const etl = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
+        const pipeline = new RedactPipeline();
+        const result = await etl.execute(pipeline, {
+            password: 'secret123',
+            secret: 'key',
+            name: 'John',
+        });
+        expect(result.password).toBeUndefined();
+        expect(result.secret).toBeUndefined();
+        expect(result.name).toBe('John');
+    });
+    it('getRedactMetadata returns options', () => {
+        const meta = (0, pii_decorator_1.getRedactMetadata)(RedactPipeline.prototype, 'redact');
+        expect(meta?.fields).toContain('password');
+    });
+    it('Mask with array shorthand', async () => {
+        let MaskArrayPipeline = class MaskArrayPipeline {
+            mask(data) {
+                return data;
+            }
+        };
+        __decorate([
+            (0, transform_decorator_1.Transform)({ step: 1, name: 'mask' }),
+            (0, pii_decorator_1.Mask)(['secret']),
+            __metadata("design:type", Function),
+            __metadata("design:paramtypes", [Object]),
+            __metadata("design:returntype", void 0)
+        ], MaskArrayPipeline.prototype, "mask", null);
+        MaskArrayPipeline = __decorate([
+            (0, pipeline_decorator_1.Pipeline)('mask-array')
+        ], MaskArrayPipeline);
+        const etl = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
+        const result = await etl.execute(new MaskArrayPipeline(), {
+            secret: 'x',
+            ok: 1,
+        });
+        expect(result.secret).toBe('****');
+        expect(result.ok).toBe(1);
+    });
+    it('Mask leaves non-object unchanged', async () => {
+        let MaskNonObjPipeline = class MaskNonObjPipeline {
+            mask(data) {
+                return data;
+            }
+        };
+        __decorate([
+            (0, transform_decorator_1.Transform)({ step: 1, name: 'mask' }),
+            (0, pii_decorator_1.Mask)(['x']),
+            __metadata("design:type", Function),
+            __metadata("design:paramtypes", [Object]),
+            __metadata("design:returntype", void 0)
+        ], MaskNonObjPipeline.prototype, "mask", null);
+        MaskNonObjPipeline = __decorate([
+            (0, pipeline_decorator_1.Pipeline)('mask-nonobj')
+        ], MaskNonObjPipeline);
+        const etl = new etl_service_1.ETLService(new schema_validator_1.SchemaValidator());
+        const result = await etl.execute(new MaskNonObjPipeline(), 'string');
+        expect(result).toBe('string');
+    });
+});

package/dist/decorators/pipeline.decorator.js

@@ -27,7 +27,7 @@ function Pipeline(nameOrOptions) {
     return (target) => {
         const options = typeof nameOrOptions === 'string' ? { name: nameOrOptions } : nameOrOptions || {};
         const metadata = {
-            name: options.name ?? target.constructor.name,
+            name: options.name ?? target.name ?? 'unknown',
             ...options,
         };
         Reflect.defineMetadata(PIPELINE_METADATA_KEY, metadata, target);

package/dist/decorators/pipeline.decorator.test.js

@@ -34,6 +34,14 @@ describe('Pipeline decorator', () => {
         ], CustomPipeline);
         expect((0, pipeline_decorator_1.getPipelineMetadata)(CustomPipeline)?.name).toBe('custom');
     });
+    it('applies metadata with no args (uses class name)', () => {
+        let DefaultPipeline = class DefaultPipeline {
+        };
+        DefaultPipeline = __decorate([
+            (0, pipeline_decorator_1.Pipeline)()
+        ], DefaultPipeline);
+        expect((0, pipeline_decorator_1.getPipelineMetadata)(DefaultPipeline)?.name).toBe('DefaultPipeline');
+    });
 });
 describe('Transform decorator', () => {
     it('applies step metadata', () => {

package/dist/decorators/transform.decorator.d.ts

@@ -1,8 +1,16 @@
 import 'reflect-metadata';
-import type { PipelineStepMetadata } from '../data.types';
+import type { PipelineStepMetadata, RetryConfig, DLQConfig } from '../data.types';
 export interface TransformOptions {
     step: number;
     name: string;
+    /** Execute step only when this predicate returns true */
+    when?: (data: unknown) => boolean;
+    /** Retry failed step with backoff */
+    retry?: RetryConfig;
+    /** Per-step execution timeout in milliseconds */
+    timeoutMs?: number;
+    /** Dead letter queue — called on failure instead of throwing */
+    dlq?: DLQConfig;
 }
 /**
  * @Transform decorator - Data transformation with step ordering

package/dist/decorators/transform.decorator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/transform.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAI1D,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,eAAe,CAapE;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,oBAAoB,GAAG,SAAS,CAElC"}
+{"version":3,"file":"transform.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/transform.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAIlF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IAClC,qCAAqC;IACrC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,eAAe,CAiBpE;AAED,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,oBAAoB,GAAG,SAAS,CAElC"}

package/dist/decorators/transform.decorator.js

@@ -26,6 +26,10 @@ function Transform(options) {
             step: options.step,
             name: options.name,
             type: 'transform',
+            when: options.when,
+            retry: options.retry,
+            timeoutMs: options.timeoutMs,
+            dlq: options.dlq,
         };
         Reflect.defineMetadata(TRANSFORM_METADATA_KEY, metadata, target, propertyKey);
         core_1.default.debug(`Transform decorator applied: ${target.constructor.name}.${String(propertyKey)} step=${options.step}`);

package/dist/decorators/validate.decorator.d.ts

@@ -1,10 +1,14 @@
 import 'reflect-metadata';
-import type { PipelineStepMetadata } from '../data.types';
+import type { PipelineStepMetadata, RetryConfig, DLQConfig } from '../data.types';
 import type { BaseSchema } from '../schema/schema';
 export interface ValidateOptions {
     step: number;
     name: string;
     schema: BaseSchema;
+    when?: (data: unknown) => boolean;
+    retry?: RetryConfig;
+    timeoutMs?: number;
+    dlq?: DLQConfig;
 }
 /**
  * @Validate decorator - Schema validation with step ordering

package/dist/decorators/validate.decorator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validate.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/validate.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAclE;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,oBAAoB,GAAG,SAAS,CAElC"}
+{"version":3,"file":"validate.decorator.d.ts","sourceRoot":"","sources":["../../src/decorators/validate.decorator.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAClF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAInD,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IAClC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAkBlE;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAAG,MAAM,GAC3B,oBAAoB,GAAG,SAAS,CAElC"}

package/dist/decorators/validate.decorator.js

@@ -34,6 +34,10 @@ function Validate(options) {
             name: options.name,
             type: 'validate',
             schema: options.schema,
+            when: options.when,
+            retry: options.retry,
+            timeoutMs: options.timeoutMs,
+            dlq: options.dlq,
         };
         Reflect.defineMetadata(VALIDATE_METADATA_KEY, metadata, target, propertyKey);
         core_1.default.debug(`Validate decorator applied: ${target.constructor.name}.${String(propertyKey)} step=${options.step}`);

package/dist/flink.service.d.ts

@@ -46,5 +46,35 @@ export declare class FlinkService {
         endTime?: number;
         duration?: number;
     }[]>;
+    /**
+     * Deploy a stream pipeline by uploading a JAR and running it.
+     * @param pipeline  The @Stream-decorated pipeline instance
+     * @param jarFile   Local path to the compiled pipeline JAR
+     * @param config    Optional Flink job config overrides
+     */
+    deployStreamWithJar(pipeline: object, jarFile: string, config?: Partial<FlinkJobConfig>): Promise<DeployStreamResult>;
+    /**
+     * Deploy a streaming pipeline using Flink SQL Gateway.
+     * @param sql       The SQL DDL+DML to submit (CREATE TABLE + INSERT INTO)
+     * @param sessionId Optional existing session ID; a new session is created if omitted
+     */
+    deployStreamWithSql(sql: string, sessionId?: string): Promise<{
+        operationId: string;
+        sessionId: string;
+    }>;
+    uploadJar(jarFile: string): Promise<string>;
+    runJar(jarId: string, options?: {
+        jobName?: string;
+        parallelism?: number;
+        entryClass?: string;
+        programArgs?: string;
+    }): Promise<string>;
+    listJars(): Promise<Array<{
+        id: string;
+        name: string;
+        uploaded: number;
+    }>>;
+    deleteJar(jarId: string): Promise<void>;
+    createSqlSession(properties?: Record<string, string>): Promise<string>;
 }
 //# sourceMappingURL=flink.service.d.ts.map

package/dist/flink.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"flink.service.d.ts","sourceRoot":"","sources":["../src/flink.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;GAGG;AACH,qBACa,YAAY;IAIrB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAJhC,OAAO,CAAC,WAAW,CAA4B;gBAG5B,UAAU,EAAE,UAAU,EACtB,aAAa,EAAE,aAAa;IAG/C,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI;IAI1C,SAAS,IAAI,WAAW;IASlB,YAAY,CAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,OAAO,CAAC,kBAAkB,CAAC;IAyBxB,YAAY,CAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9D,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIzF,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIjF,cAAc,IAAI,OAAO,CAAC;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI5E,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAIrC,QAAQ,IAAI,OAAO,CACvB;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAC1F;CAGF"}
+{"version":3,"file":"flink.service.d.ts","sourceRoot":"","sources":["../src/flink.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;GAGG;AACH,qBACa,YAAY;IAIrB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAJhC,OAAO,CAAC,WAAW,CAA4B;gBAG5B,UAAU,EAAE,UAAU,EACtB,aAAa,EAAE,aAAa;IAG/C,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI;IAI1C,SAAS,IAAI,WAAW;IASlB,YAAY,CAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,OAAO,CAAC,kBAAkB,CAAC;IAyBxB,YAAY,CAChB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9D,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIzF,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIjF,cAAc,IAAI,OAAO,CAAC;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI5E,eAAe,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAIrC,QAAQ,IAAI,OAAO,CACvB;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAC1F;IAID;;;;;OAKG;IACG,mBAAmB,CACvB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAC/B,OAAO,CAAC,kBAAkB,CAAC;IAkB9B;;;;OAIG;IACG,mBAAmB,CACvB,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAOhD,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAI3C,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACP,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACjB,GACL,OAAO,CAAC,MAAM,CAAC;IAIZ,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAI1E,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,gBAAgB,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;CAG7E"}

package/dist/flink.service.js

@@ -37,7 +37,7 @@ let FlinkService = class FlinkService {
         const { jobConfig, jobGraph } = this.streamBuilder.buildConfig(pipeline, config);
         const client = this.getClient();
         try {
-            const jobId = await client.submitJob(jobConfig, jobGraph);
+            const jobId = await client.submitJob(jobConfig);
             return {
                 jobId,
                 status: 'submitted',
@@ -47,7 +47,7 @@ let FlinkService = class FlinkService {
             };
         }
         catch {
-            // submitJob throws - return config for manual deployment
+            // submitJob throws without jarFile - return config for manual deployment
             return {
                 status: 'config_generated',
                 jobConfig,
@@ -77,6 +77,54 @@ let FlinkService = class FlinkService {
     async listJobs() {
         return this.getClient().listJobs();
     }
+    /**
+     * Deploy a stream pipeline by uploading a JAR and running it.
+     * @param pipeline  The @Stream-decorated pipeline instance
+     * @param jarFile   Local path to the compiled pipeline JAR
+     * @param config    Optional Flink job config overrides
+     */
+    async deployStreamWithJar(pipeline, jarFile, config) {
+        const { jobConfig, jobGraph } = this.streamBuilder.buildConfig(pipeline, config);
+        const client = this.getClient();
+        const jobId = await client.submitJob(jobConfig, {
+            jarFile,
+            jobName: jobConfig.jobName,
+            parallelism: jobConfig.parallelism,
+        });
+        return {
+            jobId,
+            status: 'submitted',
+            webUI: `${client.url}/#/job/${jobId}`,
+            jobConfig,
+            jobGraph,
+        };
+    }
+    /**
+     * Deploy a streaming pipeline using Flink SQL Gateway.
+     * @param sql       The SQL DDL+DML to submit (CREATE TABLE + INSERT INTO)
+     * @param sessionId Optional existing session ID; a new session is created if omitted
+     */
+    async deployStreamWithSql(sql, sessionId) {
+        const client = this.getClient();
+        const sid = sessionId ?? (await client.createSqlSession());
+        const operationId = await client.submitSql(sql, sid);
+        return { operationId, sessionId: sid };
+    }
+    async uploadJar(jarFile) {
+        return this.getClient().uploadJar(jarFile);
+    }
+    async runJar(jarId, options = {}) {
+        return this.getClient().runJar(jarId, options);
+    }
+    async listJars() {
+        return this.getClient().listJars();
+    }
+    async deleteJar(jarId) {
+        return this.getClient().deleteJar(jarId);
+    }
+    async createSqlSession(properties) {
+        return this.getClient().createSqlSession(properties);
+    }
 };
 exports.FlinkService = FlinkService;
 exports.FlinkService = FlinkService = __decorate([

package/dist/index.d.ts

@@ -4,23 +4,29 @@
 import 'reflect-metadata';
 export { DataModule, type DataModuleOptions } from './data.module';
 export { Schema } from './schema/schema';
-export type { BaseSchema, StringSchema, NumberSchema, ObjectSchema, SchemaValidationError, } from './schema/schema';
-export { Pipeline, Transform, Validate, Stream, getPipelineMetadata, hasPipelineMetadata, getTransformMetadata, getValidateMetadata, getStreamMetadata, hasStreamMetadata, type PipelineOptions, type TransformOptions, type ValidateOptions, type StreamOptions, } from './decorators';
-export { ETLService, type PipelineStep } from './pipelines/etl.service';
+export type { BaseSchema, StringSchema, NumberSchema, BooleanSchema, DateSchema, ObjectSchema, ArraySchema, LiteralSchema, UnionSchema, SchemaValidationError, SchemaValidator as SchemaValidatorFn, Infer, } from './schema/schema';
+export { Pipeline, Transform, Validate, Stream, Mask, Redact, Encrypt, Decrypt, getPipelineMetadata, hasPipelineMetadata, getTransformMetadata, getValidateMetadata, getStreamMetadata, hasStreamMetadata, getMaskMetadata, getRedactMetadata, type PipelineOptions, type TransformOptions, type ValidateOptions, type StreamOptions, type MaskOptions, type RedactOptions, type EncryptOptions, type DecryptOptions, } from './decorators';
+export { ETLService, type PipelineStep, type PipelineExecutionEvent, type PipelineEventHandler, } from './pipelines/etl.service';
 export { PipelineBase } from './pipelines/pipeline.base';
-export { PipelineBuilder, type PipelineStepConfig } from './pipelines/pipeline.builder';
+export { PipelineBuilder, type PipelineStepConfig, type PipelineDefinition, type SerializedStep, } from './pipelines/pipeline.builder';
 export { StreamService } from './pipelines/stream.service';
 export { FlinkClient } from './streaming/flink/flink.client';
 export { FlinkJob } from './streaming/flink/flink.job';
 export { mapToFlinkOperator, createFlinkJobGraph, type FlinkOperator, type FlinkOperatorType, } from './streaming/flink/flink.operators';
 export { StreamBuilder } from './streaming/stream.builder';
-export { StreamProcessor } from './streaming/stream.processor';
+export { StreamProcessor, type WindowedBatch } from './streaming/stream.processor';
 export { FlinkService, type DeployStreamResult } from './flink.service';
 export { SchemaValidator, SchemaValidationException } from './validators/schema.validator';
 export { TransformerService } from './transformers/transformer.service';
 export { trimString, toLowerCase, toUpperCase, parseJson, stringifyJson, pick, omit, renameKeys, } from './transformers/built-in.transformers';
-export { QualityService, type QualityCheckResult, type DataQualityReport, } from './quality/quality.service';
+export { QualityService, type QualityCheckResult, type DataQualityReport, type DataProfile, type FieldProfile, type AnomalyResult, } from './quality/quality.service';
+export type { DataSource, DataSink, ConnectorOptions } from './connectors/connector.interface';
+export { CsvSource, CsvSink, type CsvSourceOptions, type CsvSinkOptions, } from './connectors/csv.connector';
+export { HttpSource, HttpSink, type HttpSourceOptions, type HttpSinkOptions, } from './connectors/http.connector';
+export { MemorySource, MemorySink } from './connectors/memory.connector';
+export { TelemetryService, createPrometheusExporter, type PipelineSpan, type MetricPoint, type LineageEntry, type SpanExporter, type MetricExporter, } from './telemetry/telemetry';
+export { SchemaFaker, PipelineTestHarness, MockSource, MockSink, type StepSnapshot, type PipelineRunResult, } from './testing';
 export { Injectable } from '@hazeljs/core';
-export type { PipelineStepMetadata, StreamMetadata, FlinkJobConfig, FlinkAuthConfig, } from './data.types';
+export type { PipelineStepMetadata, StreamMetadata, FlinkJobConfig, FlinkAuthConfig, RetryConfig, DLQConfig, } from './data.types';
 export type { FlinkClientConfig, FlinkJobInfo } from './streaming/flink/flink.client';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EACV,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,MAAM,EACN,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,GACnB,MAAM,cAAc,CAAC;AAGtB,OAAO,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,KAAK,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,aAAa,EAClB,KAAK,iBAAiB,GACvB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EACL,UAAU,EACV,WAAW,EACX,WAAW,EACX,SAAS,EACT,aAAa,EACb,IAAI,EACJ,IAAI,EACJ,UAAU,GACX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,cAAc,EACd,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,GACvB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,YAAY,EACV,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EACV,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,UAAU,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,WAAW,EACX,qBAAqB,EACrB,eAAe,IAAI,iBAAiB,EACpC,KAAK,GACN,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,MAAM,EAEN,IAAI,EACJ,MAAM,EACN,OAAO,EACP,OAAO,EAEP,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EAEjB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,GACpB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,GAC1B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,cAAc,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,aAAa,EAClB,KAAK,iBAAiB,GACvB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,KAAK,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAGnF,OAAO,EAAE,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EACL,UAAU,EACV,WAAW,EACX,WAAW,EACX,SAAS,EACT,aAAa,EACb,IAAI,EACJ,IAAI,EACJ,UAAU,GACX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,cAAc,EACd,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AAC/F,OAAO,EACL,SAAS,EACT,OAAO,EACP,KAAK,gBAAgB,EACrB,KAAK,cAAc,GACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,KAAK,iBAAiB,EACtB,KAAK,eAAe,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAGzE,OAAO,EACL,gBAAgB,EAChB,wBAAwB,EACxB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,UAAU,EACV,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,iBAAiB,GACvB,MAAM,WAAW,CAAC;AAGnB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,YAAY,EACV,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,eAAe,EACf,WAAW,EACX,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC"}

package/dist/index.js

@@ -3,27 +3,36 @@
  * @hazeljs/data - Data Processing & ETL for HazelJS
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Injectable = exports.QualityService = exports.renameKeys = exports.omit = exports.pick = exports.stringifyJson = exports.parseJson = exports.toUpperCase = exports.toLowerCase = exports.trimString = exports.TransformerService = exports.SchemaValidationException = exports.SchemaValidator = exports.FlinkService = exports.StreamProcessor = exports.StreamBuilder = exports.createFlinkJobGraph = exports.mapToFlinkOperator = exports.FlinkJob = exports.FlinkClient = exports.StreamService = exports.PipelineBuilder = exports.PipelineBase = exports.ETLService = exports.hasStreamMetadata = exports.getStreamMetadata = exports.getValidateMetadata = exports.getTransformMetadata = exports.hasPipelineMetadata = exports.getPipelineMetadata = exports.Stream = exports.Validate = exports.Transform = exports.Pipeline = exports.Schema = exports.DataModule = void 0;
+exports.SchemaFaker = exports.createPrometheusExporter = exports.TelemetryService = exports.MemorySink = exports.MemorySource = exports.HttpSink = exports.HttpSource = exports.CsvSink = exports.CsvSource = exports.QualityService = exports.renameKeys = exports.omit = exports.pick = exports.stringifyJson = exports.parseJson = exports.toUpperCase = exports.toLowerCase = exports.trimString = exports.TransformerService = exports.SchemaValidationException = exports.SchemaValidator = exports.FlinkService = exports.StreamProcessor = exports.StreamBuilder = exports.createFlinkJobGraph = exports.mapToFlinkOperator = exports.FlinkJob = exports.FlinkClient = exports.StreamService = exports.PipelineBuilder = exports.PipelineBase = exports.ETLService = exports.getRedactMetadata = exports.getMaskMetadata = exports.hasStreamMetadata = exports.getStreamMetadata = exports.getValidateMetadata = exports.getTransformMetadata = exports.hasPipelineMetadata = exports.getPipelineMetadata = exports.Decrypt = exports.Encrypt = exports.Redact = exports.Mask = exports.Stream = exports.Validate = exports.Transform = exports.Pipeline = exports.Schema = exports.DataModule = void 0;
+exports.Injectable = exports.MockSink = exports.MockSource = exports.PipelineTestHarness = void 0;
 require("reflect-metadata");
-// Module
+// ─── Module ──────────────────────────────────────────────────────────────────
 var data_module_1 = require("./data.module");
 Object.defineProperty(exports, "DataModule", { enumerable: true, get: function () { return data_module_1.DataModule; } });
-// Schema
+// ─── Schema ──────────────────────────────────────────────────────────────────
 var schema_1 = require("./schema/schema");
 Object.defineProperty(exports, "Schema", { enumerable: true, get: function () { return schema_1.Schema; } });
-// Decorators
+// ─── Decorators ───────────────────────────────────────────────────────────────
 var decorators_1 = require("./decorators");
 Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return decorators_1.Pipeline; } });
 Object.defineProperty(exports, "Transform", { enumerable: true, get: function () { return decorators_1.Transform; } });
 Object.defineProperty(exports, "Validate", { enumerable: true, get: function () { return decorators_1.Validate; } });
 Object.defineProperty(exports, "Stream", { enumerable: true, get: function () { return decorators_1.Stream; } });
+// PII decorators
+Object.defineProperty(exports, "Mask", { enumerable: true, get: function () { return decorators_1.Mask; } });
+Object.defineProperty(exports, "Redact", { enumerable: true, get: function () { return decorators_1.Redact; } });
+Object.defineProperty(exports, "Encrypt", { enumerable: true, get: function () { return decorators_1.Encrypt; } });
+Object.defineProperty(exports, "Decrypt", { enumerable: true, get: function () { return decorators_1.Decrypt; } });
+// Metadata helpers
 Object.defineProperty(exports, "getPipelineMetadata", { enumerable: true, get: function () { return decorators_1.getPipelineMetadata; } });
 Object.defineProperty(exports, "hasPipelineMetadata", { enumerable: true, get: function () { return decorators_1.hasPipelineMetadata; } });
 Object.defineProperty(exports, "getTransformMetadata", { enumerable: true, get: function () { return decorators_1.getTransformMetadata; } });
 Object.defineProperty(exports, "getValidateMetadata", { enumerable: true, get: function () { return decorators_1.getValidateMetadata; } });
 Object.defineProperty(exports, "getStreamMetadata", { enumerable: true, get: function () { return decorators_1.getStreamMetadata; } });
 Object.defineProperty(exports, "hasStreamMetadata", { enumerable: true, get: function () { return decorators_1.hasStreamMetadata; } });
-// Pipelines
+Object.defineProperty(exports, "getMaskMetadata", { enumerable: true, get: function () { return decorators_1.getMaskMetadata; } });
+Object.defineProperty(exports, "getRedactMetadata", { enumerable: true, get: function () { return decorators_1.getRedactMetadata; } });
+// ─── Pipelines ────────────────────────────────────────────────────────────────
 var etl_service_1 = require("./pipelines/etl.service");
 Object.defineProperty(exports, "ETLService", { enumerable: true, get: function () { return etl_service_1.ETLService; } });
 var pipeline_base_1 = require("./pipelines/pipeline.base");
@@ -32,7 +41,7 @@ var pipeline_builder_1 = require("./pipelines/pipeline.builder");
 Object.defineProperty(exports, "PipelineBuilder", { enumerable: true, get: function () { return pipeline_builder_1.PipelineBuilder; } });
 var stream_service_1 = require("./pipelines/stream.service");
 Object.defineProperty(exports, "StreamService", { enumerable: true, get: function () { return stream_service_1.StreamService; } });
-// Streaming
+// ─── Streaming ────────────────────────────────────────────────────────────────
 var flink_client_1 = require("./streaming/flink/flink.client");
 Object.defineProperty(exports, "FlinkClient", { enumerable: true, get: function () { return flink_client_1.FlinkClient; } });
 var flink_job_1 = require("./streaming/flink/flink.job");
@@ -44,7 +53,7 @@ var stream_builder_1 = require("./streaming/stream.builder");
 Object.defineProperty(exports, "StreamBuilder", { enumerable: true, get: function () { return stream_builder_1.StreamBuilder; } });
 var stream_processor_1 = require("./streaming/stream.processor");
 Object.defineProperty(exports, "StreamProcessor", { enumerable: true, get: function () { return stream_processor_1.StreamProcessor; } });
-// Services
+// ─── Services ─────────────────────────────────────────────────────────────────
 var flink_service_1 = require("./flink.service");
 Object.defineProperty(exports, "FlinkService", { enumerable: true, get: function () { return flink_service_1.FlinkService; } });
 var schema_validator_1 = require("./validators/schema.validator");
@@ -63,6 +72,25 @@ Object.defineProperty(exports, "omit", { enumerable: true, get: function () { re
 Object.defineProperty(exports, "renameKeys", { enumerable: true, get: function () { return built_in_transformers_1.renameKeys; } });
 var quality_service_1 = require("./quality/quality.service");
 Object.defineProperty(exports, "QualityService", { enumerable: true, get: function () { return quality_service_1.QualityService; } });
-// Re-export Injectable
+var csv_connector_1 = require("./connectors/csv.connector");
+Object.defineProperty(exports, "CsvSource", { enumerable: true, get: function () { return csv_connector_1.CsvSource; } });
+Object.defineProperty(exports, "CsvSink", { enumerable: true, get: function () { return csv_connector_1.CsvSink; } });
+var http_connector_1 = require("./connectors/http.connector");
+Object.defineProperty(exports, "HttpSource", { enumerable: true, get: function () { return http_connector_1.HttpSource; } });
+Object.defineProperty(exports, "HttpSink", { enumerable: true, get: function () { return http_connector_1.HttpSink; } });
+var memory_connector_1 = require("./connectors/memory.connector");
+Object.defineProperty(exports, "MemorySource", { enumerable: true, get: function () { return memory_connector_1.MemorySource; } });
+Object.defineProperty(exports, "MemorySink", { enumerable: true, get: function () { return memory_connector_1.MemorySink; } });
+// ─── Telemetry ────────────────────────────────────────────────────────────────
+var telemetry_1 = require("./telemetry/telemetry");
+Object.defineProperty(exports, "TelemetryService", { enumerable: true, get: function () { return telemetry_1.TelemetryService; } });
+Object.defineProperty(exports, "createPrometheusExporter", { enumerable: true, get: function () { return telemetry_1.createPrometheusExporter; } });
+// ─── Testing Utilities ────────────────────────────────────────────────────────
+var testing_1 = require("./testing");
+Object.defineProperty(exports, "SchemaFaker", { enumerable: true, get: function () { return testing_1.SchemaFaker; } });
+Object.defineProperty(exports, "PipelineTestHarness", { enumerable: true, get: function () { return testing_1.PipelineTestHarness; } });
+Object.defineProperty(exports, "MockSource", { enumerable: true, get: function () { return testing_1.MockSource; } });
+Object.defineProperty(exports, "MockSink", { enumerable: true, get: function () { return testing_1.MockSink; } });
+// ─── Re-exports ───────────────────────────────────────────────────────────────
 var core_1 = require("@hazeljs/core");
 Object.defineProperty(exports, "Injectable", { enumerable: true, get: function () { return core_1.Injectable; } });

package/dist/pipelines/etl.service.d.ts

@@ -1,20 +1,59 @@
 import { SchemaValidator } from '../validators/schema.validator';
 import type { BaseSchema } from '../schema/schema';
+import type { RetryConfig, DLQConfig } from '../data.types';
 export interface PipelineStep {
     step: number;
     name: string;
     type: 'transform' | 'validate';
     method: string;
     schema?: BaseSchema;
+    when?: (data: unknown) => boolean;
+    retry?: RetryConfig;
+    timeoutMs?: number;
+    dlq?: DLQConfig;
 }
+export interface PipelineExecutionEvent {
+    pipeline: string;
+    step: number;
+    stepName: string;
+    durationMs: number;
+    success: boolean;
+    skipped?: boolean;
+    error?: string;
+}
+export type PipelineEventHandler = (event: PipelineExecutionEvent) => void;
 /**
- * ETL Service - Orchestrates pipeline execution
- * Executes steps sequentially: output from step N → input to step N+1
+ * ETL Service — orchestrates pipeline execution.
+ *
+ * Features:
+ * - Sequential step execution (output N → input N+1)
+ * - Conditional steps via `when` predicate
+ * - Per-step retry with fixed/exponential backoff
+ * - Per-step execution timeout
+ * - Dead letter queue (DLQ) for graceful failure handling
+ * - Pipeline event hooks for observability
  */
 export declare class ETLService {
     private readonly schemaValidator;
+    private eventHandlers;
     constructor(schemaValidator: SchemaValidator);
+    /** Register a handler called after each step completes or fails. */
+    onStepComplete(handler: PipelineEventHandler): void;
+    private emit;
     extractSteps(instance: object): PipelineStep[];
     execute<T = unknown>(pipelineInstance: object, input: unknown): Promise<T>;
+    /**
+     * Execute multiple items through the pipeline in parallel.
+     * Items that fail are routed to the DLQ if configured, otherwise they propagate.
+     */
+    executeBatch<T = unknown>(pipelineInstance: object, items: unknown[], options?: {
+        concurrency?: number;
+    }): Promise<{
+        results: T[];
+        errors: Array<{
+            item: unknown;
+            error: Error;
+        }>;
+    }>;
 }
 //# sourceMappingURL=etl.service.d.ts.map

package/dist/pipelines/etl.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"etl.service.d.ts","sourceRoot":"","sources":["../../src/pipelines/etl.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAGnD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;GAGG;AACH,qBACa,UAAU;IACT,OAAO,CAAC,QAAQ,CAAC,eAAe;gBAAf,eAAe,EAAE,eAAe;IAE7D,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,EAAE;IAiCxC,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;CA6BjF"}
+{"version":3,"file":"etl.service.d.ts","sourceRoot":"","sources":["../../src/pipelines/etl.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAG5D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IAClC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;AAiD3E;;;;;;;;;;GAUG;AACH,qBACa,UAAU;IAGT,OAAO,CAAC,QAAQ,CAAC,eAAe;IAF5C,OAAO,CAAC,aAAa,CAA8B;gBAEtB,eAAe,EAAE,eAAe;IAE7D,oEAAoE;IACpE,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAInD,OAAO,CAAC,IAAI;IAUZ,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,EAAE;IAyCxC,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAkFhF;;;OAGG;IACG,YAAY,CAAC,CAAC,GAAG,OAAO,EAC5B,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,OAAO,EAAE,EAChB,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACrC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAAC,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,OAAO,CAAC;YAAC,KAAK,EAAE,KAAK,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAyB7E"}