@hazeljs/data 0.2.0-alpha.1

package/dist/index.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @hazeljs/data - Data Processing & ETL for HazelJS
+ */
+import 'reflect-metadata';
+export { DataModule, type DataModuleOptions } from './data.module';
+export { Schema } from './schema/schema';
+export type { BaseSchema, StringSchema, NumberSchema, BooleanSchema, DateSchema, ObjectSchema, ArraySchema, LiteralSchema, UnionSchema, SchemaValidationError, SchemaValidator as SchemaValidatorFn, Infer, } from './schema/schema';
+export { Pipeline, Transform, Validate, Stream, Mask, Redact, Encrypt, Decrypt, getPipelineMetadata, hasPipelineMetadata, getTransformMetadata, getValidateMetadata, getStreamMetadata, hasStreamMetadata, getMaskMetadata, getRedactMetadata, type PipelineOptions, type TransformOptions, type ValidateOptions, type StreamOptions, type MaskOptions, type RedactOptions, type EncryptOptions, type DecryptOptions, } from './decorators';
+export { ETLService, type PipelineStep, type PipelineExecutionEvent, type PipelineEventHandler, } from './pipelines/etl.service';
+export { PipelineBase } from './pipelines/pipeline.base';
+export { PipelineBuilder, type PipelineStepConfig, type PipelineDefinition, type SerializedStep, } from './pipelines/pipeline.builder';
+export { StreamService } from './pipelines/stream.service';
+export { FlinkClient } from './streaming/flink/flink.client';
+export { FlinkJob } from './streaming/flink/flink.job';
+export { mapToFlinkOperator, createFlinkJobGraph, type FlinkOperator, type FlinkOperatorType, } from './streaming/flink/flink.operators';
+export { StreamBuilder } from './streaming/stream.builder';
+export { StreamProcessor, type WindowedBatch } from './streaming/stream.processor';
+export { FlinkService, type DeployStreamResult } from './flink.service';
+export { SchemaValidator, SchemaValidationException } from './validators/schema.validator';
+export { TransformerService } from './transformers/transformer.service';
+export { trimString, toLowerCase, toUpperCase, parseJson, stringifyJson, pick, omit, renameKeys, } from './transformers/built-in.transformers';
+export { QualityService, type QualityCheckResult, type DataQualityReport, type DataProfile, type FieldProfile, type AnomalyResult, } from './quality/quality.service';
+export type { DataSource, DataSink, ConnectorOptions } from './connectors/connector.interface';
+export { CsvSource, CsvSink, type CsvSourceOptions, type CsvSinkOptions, } from './connectors/csv.connector';
+export { HttpSource, HttpSink, type HttpSourceOptions, type HttpSinkOptions, } from './connectors/http.connector';
+export { MemorySource, MemorySink } from './connectors/memory.connector';
+export { TelemetryService, createPrometheusExporter, type PipelineSpan, type MetricPoint, type LineageEntry, type SpanExporter, type MetricExporter, } from './telemetry/telemetry';
+export { SchemaFaker, PipelineTestHarness, MockSource, MockSink, type StepSnapshot, type PipelineRunResult, } from './testing';
+export { Injectable } from '@hazeljs/core';
+export type { PipelineStepMetadata, StreamMetadata, FlinkJobConfig, FlinkAuthConfig, RetryConfig, DLQConfig, } from './data.types';
+export type { FlinkClientConfig, FlinkJobInfo } from './streaming/flink/flink.client';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGnE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EACV,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,UAAU,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,WAAW,EACX,qBAAqB,EACrB,eAAe,IAAI,iBAAiB,EACpC,KAAK,GACN,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,MAAM,EAEN,IAAI,EACJ,MAAM,EACN,OAAO,EACP,OAAO,EAEP,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EAEjB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,GACpB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,GAC1B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EACL,eAAe,EACf,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,cAAc,GACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,aAAa,EAClB,KAAK,iBAAiB,GACvB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,KAAK,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAGnF,OAAO,EAAE,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EACL,UAAU,EACV,WAAW,EACX,WAAW,EACX,SAAS,EACT,aAAa,EACb,IAAI,EACJ,IAAI,EACJ,UAAU,GACX,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,cAAc,EACd,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,aAAa,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AAC/F,OAAO,EACL,SAAS,EACT,OAAO,EACP,KAAK,gBAAgB,EACrB,KAAK,cAAc,GACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,KAAK,iBAAiB,EACtB,KAAK,eAAe,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAGzE,OAAO,EACL,gBAAgB,EAChB,wBAAwB,EACxB,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,UAAU,EACV,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,iBAAiB,GACvB,MAAM,WAAW,CAAC;AAGnB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAG3C,YAAY,EACV,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,eAAe,EACf,WAAW,EACX,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC"}

package/dist/index.js

@@ -0,0 +1,96 @@
+"use strict";
+/**
+ * @hazeljs/data - Data Processing & ETL for HazelJS
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SchemaFaker = exports.createPrometheusExporter = exports.TelemetryService = exports.MemorySink = exports.MemorySource = exports.HttpSink = exports.HttpSource = exports.CsvSink = exports.CsvSource = exports.QualityService = exports.renameKeys = exports.omit = exports.pick = exports.stringifyJson = exports.parseJson = exports.toUpperCase = exports.toLowerCase = exports.trimString = exports.TransformerService = exports.SchemaValidationException = exports.SchemaValidator = exports.FlinkService = exports.StreamProcessor = exports.StreamBuilder = exports.createFlinkJobGraph = exports.mapToFlinkOperator = exports.FlinkJob = exports.FlinkClient = exports.StreamService = exports.PipelineBuilder = exports.PipelineBase = exports.ETLService = exports.getRedactMetadata = exports.getMaskMetadata = exports.hasStreamMetadata = exports.getStreamMetadata = exports.getValidateMetadata = exports.getTransformMetadata = exports.hasPipelineMetadata = exports.getPipelineMetadata = exports.Decrypt = exports.Encrypt = exports.Redact = exports.Mask = exports.Stream = exports.Validate = exports.Transform = exports.Pipeline = exports.Schema = exports.DataModule = void 0;
+exports.Injectable = exports.MockSink = exports.MockSource = exports.PipelineTestHarness = void 0;
+require("reflect-metadata");
+// ─── Module ──────────────────────────────────────────────────────────────────
+var data_module_1 = require("./data.module");
+Object.defineProperty(exports, "DataModule", { enumerable: true, get: function () { return data_module_1.DataModule; } });
+// ─── Schema ──────────────────────────────────────────────────────────────────
+var schema_1 = require("./schema/schema");
+Object.defineProperty(exports, "Schema", { enumerable: true, get: function () { return schema_1.Schema; } });
+// ─── Decorators ───────────────────────────────────────────────────────────────
+var decorators_1 = require("./decorators");
+Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return decorators_1.Pipeline; } });
+Object.defineProperty(exports, "Transform", { enumerable: true, get: function () { return decorators_1.Transform; } });
+Object.defineProperty(exports, "Validate", { enumerable: true, get: function () { return decorators_1.Validate; } });
+Object.defineProperty(exports, "Stream", { enumerable: true, get: function () { return decorators_1.Stream; } });
+// PII decorators
+Object.defineProperty(exports, "Mask", { enumerable: true, get: function () { return decorators_1.Mask; } });
+Object.defineProperty(exports, "Redact", { enumerable: true, get: function () { return decorators_1.Redact; } });
+Object.defineProperty(exports, "Encrypt", { enumerable: true, get: function () { return decorators_1.Encrypt; } });
+Object.defineProperty(exports, "Decrypt", { enumerable: true, get: function () { return decorators_1.Decrypt; } });
+// Metadata helpers
+Object.defineProperty(exports, "getPipelineMetadata", { enumerable: true, get: function () { return decorators_1.getPipelineMetadata; } });
+Object.defineProperty(exports, "hasPipelineMetadata", { enumerable: true, get: function () { return decorators_1.hasPipelineMetadata; } });
+Object.defineProperty(exports, "getTransformMetadata", { enumerable: true, get: function () { return decorators_1.getTransformMetadata; } });
+Object.defineProperty(exports, "getValidateMetadata", { enumerable: true, get: function () { return decorators_1.getValidateMetadata; } });
+Object.defineProperty(exports, "getStreamMetadata", { enumerable: true, get: function () { return decorators_1.getStreamMetadata; } });
+Object.defineProperty(exports, "hasStreamMetadata", { enumerable: true, get: function () { return decorators_1.hasStreamMetadata; } });
+Object.defineProperty(exports, "getMaskMetadata", { enumerable: true, get: function () { return decorators_1.getMaskMetadata; } });
+Object.defineProperty(exports, "getRedactMetadata", { enumerable: true, get: function () { return decorators_1.getRedactMetadata; } });
+// ─── Pipelines ────────────────────────────────────────────────────────────────
+var etl_service_1 = require("./pipelines/etl.service");
+Object.defineProperty(exports, "ETLService", { enumerable: true, get: function () { return etl_service_1.ETLService; } });
+var pipeline_base_1 = require("./pipelines/pipeline.base");
+Object.defineProperty(exports, "PipelineBase", { enumerable: true, get: function () { return pipeline_base_1.PipelineBase; } });
+var pipeline_builder_1 = require("./pipelines/pipeline.builder");
+Object.defineProperty(exports, "PipelineBuilder", { enumerable: true, get: function () { return pipeline_builder_1.PipelineBuilder; } });
+var stream_service_1 = require("./pipelines/stream.service");
+Object.defineProperty(exports, "StreamService", { enumerable: true, get: function () { return stream_service_1.StreamService; } });
+// ─── Streaming ────────────────────────────────────────────────────────────────
+var flink_client_1 = require("./streaming/flink/flink.client");
+Object.defineProperty(exports, "FlinkClient", { enumerable: true, get: function () { return flink_client_1.FlinkClient; } });
+var flink_job_1 = require("./streaming/flink/flink.job");
+Object.defineProperty(exports, "FlinkJob", { enumerable: true, get: function () { return flink_job_1.FlinkJob; } });
+var flink_operators_1 = require("./streaming/flink/flink.operators");
+Object.defineProperty(exports, "mapToFlinkOperator", { enumerable: true, get: function () { return flink_operators_1.mapToFlinkOperator; } });
+Object.defineProperty(exports, "createFlinkJobGraph", { enumerable: true, get: function () { return flink_operators_1.createFlinkJobGraph; } });
+var stream_builder_1 = require("./streaming/stream.builder");
+Object.defineProperty(exports, "StreamBuilder", { enumerable: true, get: function () { return stream_builder_1.StreamBuilder; } });
+var stream_processor_1 = require("./streaming/stream.processor");
+Object.defineProperty(exports, "StreamProcessor", { enumerable: true, get: function () { return stream_processor_1.StreamProcessor; } });
+// ─── Services ─────────────────────────────────────────────────────────────────
+var flink_service_1 = require("./flink.service");
+Object.defineProperty(exports, "FlinkService", { enumerable: true, get: function () { return flink_service_1.FlinkService; } });
+var schema_validator_1 = require("./validators/schema.validator");
+Object.defineProperty(exports, "SchemaValidator", { enumerable: true, get: function () { return schema_validator_1.SchemaValidator; } });
+Object.defineProperty(exports, "SchemaValidationException", { enumerable: true, get: function () { return schema_validator_1.SchemaValidationException; } });
+var transformer_service_1 = require("./transformers/transformer.service");
+Object.defineProperty(exports, "TransformerService", { enumerable: true, get: function () { return transformer_service_1.TransformerService; } });
+var built_in_transformers_1 = require("./transformers/built-in.transformers");
+Object.defineProperty(exports, "trimString", { enumerable: true, get: function () { return built_in_transformers_1.trimString; } });
+Object.defineProperty(exports, "toLowerCase", { enumerable: true, get: function () { return built_in_transformers_1.toLowerCase; } });
+Object.defineProperty(exports, "toUpperCase", { enumerable: true, get: function () { return built_in_transformers_1.toUpperCase; } });
+Object.defineProperty(exports, "parseJson", { enumerable: true, get: function () { return built_in_transformers_1.parseJson; } });
+Object.defineProperty(exports, "stringifyJson", { enumerable: true, get: function () { return built_in_transformers_1.stringifyJson; } });
+Object.defineProperty(exports, "pick", { enumerable: true, get: function () { return built_in_transformers_1.pick; } });
+Object.defineProperty(exports, "omit", { enumerable: true, get: function () { return built_in_transformers_1.omit; } });
+Object.defineProperty(exports, "renameKeys", { enumerable: true, get: function () { return built_in_transformers_1.renameKeys; } });
+var quality_service_1 = require("./quality/quality.service");
+Object.defineProperty(exports, "QualityService", { enumerable: true, get: function () { return quality_service_1.QualityService; } });
+var csv_connector_1 = require("./connectors/csv.connector");
+Object.defineProperty(exports, "CsvSource", { enumerable: true, get: function () { return csv_connector_1.CsvSource; } });
+Object.defineProperty(exports, "CsvSink", { enumerable: true, get: function () { return csv_connector_1.CsvSink; } });
+var http_connector_1 = require("./connectors/http.connector");
+Object.defineProperty(exports, "HttpSource", { enumerable: true, get: function () { return http_connector_1.HttpSource; } });
+Object.defineProperty(exports, "HttpSink", { enumerable: true, get: function () { return http_connector_1.HttpSink; } });
+var memory_connector_1 = require("./connectors/memory.connector");
+Object.defineProperty(exports, "MemorySource", { enumerable: true, get: function () { return memory_connector_1.MemorySource; } });
+Object.defineProperty(exports, "MemorySink", { enumerable: true, get: function () { return memory_connector_1.MemorySink; } });
+// ─── Telemetry ────────────────────────────────────────────────────────────────
+var telemetry_1 = require("./telemetry/telemetry");
+Object.defineProperty(exports, "TelemetryService", { enumerable: true, get: function () { return telemetry_1.TelemetryService; } });
+Object.defineProperty(exports, "createPrometheusExporter", { enumerable: true, get: function () { return telemetry_1.createPrometheusExporter; } });
+// ─── Testing Utilities ────────────────────────────────────────────────────────
+var testing_1 = require("./testing");
+Object.defineProperty(exports, "SchemaFaker", { enumerable: true, get: function () { return testing_1.SchemaFaker; } });
+Object.defineProperty(exports, "PipelineTestHarness", { enumerable: true, get: function () { return testing_1.PipelineTestHarness; } });
+Object.defineProperty(exports, "MockSource", { enumerable: true, get: function () { return testing_1.MockSource; } });
+Object.defineProperty(exports, "MockSink", { enumerable: true, get: function () { return testing_1.MockSink; } });
+// ─── Re-exports ───────────────────────────────────────────────────────────────
+var core_1 = require("@hazeljs/core");
+Object.defineProperty(exports, "Injectable", { enumerable: true, get: function () { return core_1.Injectable; } });

package/dist/pipelines/etl.service.d.ts

@@ -0,0 +1,59 @@
+import { SchemaValidator } from '../validators/schema.validator';
+import type { BaseSchema } from '../schema/schema';
+import type { RetryConfig, DLQConfig } from '../data.types';
+export interface PipelineStep {
+    step: number;
+    name: string;
+    type: 'transform' | 'validate';
+    method: string;
+    schema?: BaseSchema;
+    when?: (data: unknown) => boolean;
+    retry?: RetryConfig;
+    timeoutMs?: number;
+    dlq?: DLQConfig;
+}
+export interface PipelineExecutionEvent {
+    pipeline: string;
+    step: number;
+    stepName: string;
+    durationMs: number;
+    success: boolean;
+    skipped?: boolean;
+    error?: string;
+}
+export type PipelineEventHandler = (event: PipelineExecutionEvent) => void;
+/**
+ * ETL Service — orchestrates pipeline execution.
+ *
+ * Features:
+ * - Sequential step execution (output N → input N+1)
+ * - Conditional steps via `when` predicate
+ * - Per-step retry with fixed/exponential backoff
+ * - Per-step execution timeout
+ * - Dead letter queue (DLQ) for graceful failure handling
+ * - Pipeline event hooks for observability
+ */
+export declare class ETLService {
+    private readonly schemaValidator;
+    private eventHandlers;
+    constructor(schemaValidator: SchemaValidator);
+    /** Register a handler called after each step completes or fails. */
+    onStepComplete(handler: PipelineEventHandler): void;
+    private emit;
+    extractSteps(instance: object): PipelineStep[];
+    execute<T = unknown>(pipelineInstance: object, input: unknown): Promise<T>;
+    /**
+     * Execute multiple items through the pipeline in parallel.
+     * Items that fail are routed to the DLQ if configured, otherwise they propagate.
+     */
+    executeBatch<T = unknown>(pipelineInstance: object, items: unknown[], options?: {
+        concurrency?: number;
+    }): Promise<{
+        results: T[];
+        errors: Array<{
+            item: unknown;
+            error: Error;
+        }>;
+    }>;
+}
+//# sourceMappingURL=etl.service.d.ts.map

package/dist/pipelines/etl.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"etl.service.d.ts","sourceRoot":"","sources":["../../src/pipelines/etl.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAG5D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;IAClC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAC;AAiD3E;;;;;;;;;;GAUG;AACH,qBACa,UAAU;IAGT,OAAO,CAAC,QAAQ,CAAC,eAAe;IAF5C,OAAO,CAAC,aAAa,CAA8B;gBAEtB,eAAe,EAAE,eAAe;IAE7D,oEAAoE;IACpE,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAInD,OAAO,CAAC,IAAI;IAUZ,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,EAAE;IAyCxC,OAAO,CAAC,CAAC,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;IAkFhF;;;OAGG;IACG,YAAY,CAAC,CAAC,GAAG,OAAO,EAC5B,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,OAAO,EAAE,EAChB,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACrC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAAC,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,OAAO,CAAC;YAAC,KAAK,EAAE,KAAK,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAyB7E"}

package/dist/pipelines/etl.service.js

@@ -0,0 +1,223 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ETLService = void 0;
+const core_1 = require("@hazeljs/core");
+const decorators_1 = require("../decorators");
+const schema_validator_1 = require("../validators/schema.validator");
+const core_2 = __importDefault(require("@hazeljs/core"));
+/** Runs a promise with a per-call timeout. */
+async function withTimeout(promise, timeoutMs, stepName) {
+    return new Promise((resolve, reject) => {
+        const id = setTimeout(() => reject(new Error(`Step "${stepName}" timed out after ${timeoutMs}ms`)), timeoutMs);
+        promise.then((v) => {
+            clearTimeout(id);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(id);
+            reject(e);
+        });
+    });
+}
+/** Executes fn with retry + exponential/fixed backoff. */
+async function withRetry(fn, retry, stepName) {
+    const { attempts, delay = 500, backoff = 'fixed' } = retry;
+    let lastError = new Error('Unknown error');
+    for (let attempt = 1; attempt <= attempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+            if (attempt < attempts) {
+                const wait = backoff === 'exponential' ? delay * Math.pow(2, attempt - 1) : delay;
+                core_2.default.debug(`Step "${stepName}" attempt ${attempt} failed. Retrying in ${wait}ms...`);
+                await new Promise((r) => setTimeout(r, wait));
+            }
+        }
+    }
+    throw lastError;
+}
+/**
+ * ETL Service — orchestrates pipeline execution.
+ *
+ * Features:
+ * - Sequential step execution (output N → input N+1)
+ * - Conditional steps via `when` predicate
+ * - Per-step retry with fixed/exponential backoff
+ * - Per-step execution timeout
+ * - Dead letter queue (DLQ) for graceful failure handling
+ * - Pipeline event hooks for observability
+ */
+let ETLService = class ETLService {
+    constructor(schemaValidator) {
+        this.schemaValidator = schemaValidator;
+        this.eventHandlers = [];
+    }
+    /** Register a handler called after each step completes or fails. */
+    onStepComplete(handler) {
+        this.eventHandlers.push(handler);
+    }
+    emit(event) {
+        for (const h of this.eventHandlers) {
+            try {
+                h(event);
+            }
+            catch {
+                /* noop */
+            }
+        }
+    }
+    extractSteps(instance) {
+        const steps = [];
+        const proto = Object.getPrototypeOf(instance);
+        for (const key of Object.getOwnPropertyNames(proto)) {
+            if (key === 'constructor')
+                continue;
+            const descriptor = Object.getOwnPropertyDescriptor(proto, key);
+            if (descriptor?.value && typeof descriptor.value === 'function') {
+                const transformMeta = (0, decorators_1.getTransformMetadata)(proto, key);
+                const validateMeta = (0, decorators_1.getValidateMetadata)(proto, key);
+                if (transformMeta) {
+                    steps.push({
+                        step: transformMeta.step,
+                        name: transformMeta.name,
+                        type: 'transform',
+                        method: key,
+                        when: transformMeta.when,
+                        retry: transformMeta.retry,
+                        timeoutMs: transformMeta.timeoutMs,
+                        dlq: transformMeta.dlq,
+                    });
+                }
+                else if (validateMeta) {
+                    steps.push({
+                        step: validateMeta.step,
+                        name: validateMeta.name,
+                        type: 'validate',
+                        method: key,
+                        schema: validateMeta.schema,
+                        when: validateMeta.when,
+                        retry: validateMeta.retry,
+                        timeoutMs: validateMeta.timeoutMs,
+                        dlq: validateMeta.dlq,
+                    });
+                }
+            }
+        }
+        return steps.sort((a, b) => a.step - b.step);
+    }
+    async execute(pipelineInstance, input) {
+        const metadata = (0, decorators_1.getPipelineMetadata)(pipelineInstance.constructor);
+        const steps = this.extractSteps(pipelineInstance);
+        if (steps.length === 0) {
+            throw new Error(`Pipeline ${metadata?.name ?? 'unknown'} has no steps`);
+        }
+        core_2.default.debug(`Executing pipeline ${metadata?.name} with ${steps.length} steps`);
+        let data = input;
+        const instance = pipelineInstance;
+        for (const step of steps) {
+            const fn = instance[step.method];
+            if (typeof fn !== 'function') {
+                throw new Error(`Step ${step.name} method ${step.method} not found`);
+            }
+            // Conditional skip
+            if (step.when && !step.when(data)) {
+                core_2.default.debug(`Step "${step.name}" skipped (when predicate returned false)`);
+                this.emit({
+                    pipeline: metadata?.name ?? 'unknown',
+                    step: step.step,
+                    stepName: step.name,
+                    durationMs: 0,
+                    success: true,
+                    skipped: true,
+                });
+                continue;
+            }
+            const t0 = Date.now();
+            const runStep = async () => {
+                if (step.type === 'validate' && step.schema) {
+                    data = this.schemaValidator.validate(step.schema, data);
+                }
+                const result = fn.call(pipelineInstance, data);
+                return result instanceof Promise ? await result : result;
+            };
+            try {
+                let stepPromise = step.retry ? withRetry(runStep, step.retry, step.name) : runStep();
+                if (step.timeoutMs) {
+                    stepPromise = withTimeout(stepPromise, step.timeoutMs, step.name);
+                }
+                data = await stepPromise;
+                this.emit({
+                    pipeline: metadata?.name ?? 'unknown',
+                    step: step.step,
+                    stepName: step.name,
+                    durationMs: Date.now() - t0,
+                    success: true,
+                });
+            }
+            catch (err) {
+                const error = err instanceof Error ? err : new Error(String(err));
+                this.emit({
+                    pipeline: metadata?.name ?? 'unknown',
+                    step: step.step,
+                    stepName: step.name,
+                    durationMs: Date.now() - t0,
+                    success: false,
+                    error: error.message,
+                });
+                if (step.dlq) {
+                    core_2.default.debug(`Step "${step.name}" failed — routing to DLQ`);
+                    await Promise.resolve(step.dlq.handler(data, error, step.name));
+                    // Continue with unchanged data when routed to DLQ
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        return data;
+    }
+    /**
+     * Execute multiple items through the pipeline in parallel.
+     * Items that fail are routed to the DLQ if configured, otherwise they propagate.
+     */
+    async executeBatch(pipelineInstance, items, options = {}) {
+        const { concurrency = 10 } = options;
+        const results = [];
+        const errors = [];
+        for (let i = 0; i < items.length; i += concurrency) {
+            const batch = items.slice(i, i + concurrency);
+            const settled = await Promise.allSettled(batch.map((item) => this.execute(pipelineInstance, item)));
+            for (let j = 0; j < settled.length; j++) {
+                const s = settled[j];
+                if (s.status === 'fulfilled') {
+                    results.push(s.value);
+                }
+                else {
+                    errors.push({
+                        item: batch[j],
+                        error: s.reason instanceof Error ? s.reason : new Error(String(s.reason)),
+                    });
+                }
+            }
+        }
+        return { results, errors };
+    }
+};
+exports.ETLService = ETLService;
+exports.ETLService = ETLService = __decorate([
+    (0, core_1.Service)(),
+    __metadata("design:paramtypes", [schema_validator_1.SchemaValidator])
+], ETLService);

package/dist/pipelines/etl.service.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=etl.service.test.d.ts.map

package/dist/pipelines/etl.service.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"etl.service.test.d.ts","sourceRoot":"","sources":["../../src/pipelines/etl.service.test.ts"],"names":[],"mappings":""}