@hazeljs/data 0.2.0-beta.68 → 0.2.0-beta.70

package/README.md

@@ -1,6 +1,6 @@
 # @hazeljs/data
 
-Data Processing & ETL for HazelJS - pipelines, schema validation, streaming, and data quality.
+Data Processing & ETL for HazelJS - pipelines, schema validation, streaming, data quality, and more.
 
 [![npm version](https://img.shields.io/npm/v/@hazeljs/data.svg)](https://www.npmjs.com/package/@hazeljs/data)
 [![npm downloads](https://img.shields.io/npm/dm/@hazeljs/data)](https://www.npmjs.com/package/@hazeljs/data)
@@ -9,11 +9,16 @@ Data Processing & ETL for HazelJS - pipelines, schema validation, streaming, and
 ## Features
 
 - **Pipelines** – Declarative ETL with `@Pipeline`, `@Transform`, `@Validate` decorators
-- **Schema validation** – Fluent Schema API (string, number, object, array, email, oneOf)
-- **ETL service** – Execute multi-step pipelines with ordering and error handling
-- **Stream processing** – StreamBuilder, StreamProcessor for batch and streaming workloads
-- **Built-in transformers** – trimString, toLowerCase, parseJson, pick, omit, renameKeys
-- **Data quality** – QualityService for completeness, notNull, and custom checks
+- **Schema validation** – Fluent Schema API (string, number, boolean, date, object, array, literal, union) with `.optional()`, `.nullable()`, `.default()`, `.transform()`, `.refine()`, `Infer<T>`, `.toJsonSchema()`
+- **Pipeline options** – Conditional steps (`when`), per-step retry, timeout, dead letter queue (DLQ)
+- **PipelineBuilder** – Programmatic pipelines with `.branch()`, `.parallel()`, `.catch()`, `.toSchema()`
+- **ETL service** – Execute multi-step pipelines with `executeBatch`, `onStepComplete`
+- **Stream processing** – StreamService, StreamProcessor with tumbling/sliding/session windows and stream join
+- **Built-in transformers** – trimString, toLowerCase, toUpperCase, parseJson, stringifyJson, pick, omit, renameKeys
+- **Data quality** – QualityService with completeness, notNull, uniqueness, range, pattern, referentialIntegrity, profile(), detectAnomalies()
+- **Connectors** – DataSource/DataSink (MemorySource, MemorySink, CsvSource, HttpSource)
+- **PII decorators** – @Mask, @Redact, @Encrypt, @Decrypt for sensitive data
+- **Test utilities** – SchemaFaker, PipelineTestHarness, MockSource, MockSink
 - **Flink integration** – Optional Apache Flink deployment for distributed stream processing
 
 ## Installation
@@ -48,18 +53,24 @@ import {
   Validate,
   ETLService,
   Schema,
+  Infer,
 } from '@hazeljs/data';
 
-const OrderSchema = Schema.object()
-  .prop('id', Schema.string().required())
-  .prop('customerId', Schema.string().required())
-  .prop('status', Schema.string().oneOf(['pending', 'paid', 'shipped', 'delivered', 'cancelled']))
-  .prop('items', Schema.array().items(Schema.object()
-    .prop('sku', Schema.string().minLength(1))
-    .prop('qty', Schema.number().min(1))
-    .prop('price', Schema.number().min(0))
-  ))
-  .required();
+const OrderSchema = Schema.object({
+  id: Schema.string().min(1),
+  customerId: Schema.string().min(1),
+  items: Schema.array(
+    Schema.object({
+      sku: Schema.string().min(1),
+      qty: Schema.number().min(1),
+      price: Schema.number().min(0),
+    })
+  ),
+  status: Schema.string().oneOf(['pending', 'paid', 'shipped', 'delivered', 'cancelled']),
+  createdAt: Schema.string().min(1),
+});
+
+type Order = Infer<typeof OrderSchema>;
 
 @Pipeline('order-processing')
 @Injectable()
@@ -69,30 +80,21 @@ export class OrderProcessingPipeline extends PipelineBase {
   }
 
   @Transform({ step: 1, name: 'normalize' })
-  async normalize(data: Record<string, unknown>): Promise<Record<string, unknown>> {
-    return {
-      ...data,
-      status: String(data.status).toLowerCase(),
-    };
+  async normalize(data: unknown): Promise<Order> {
+    return { ...(data as Order), status: String((data as Order).status).toLowerCase() };
   }
 
-  @Validate({ step: 2, schema: OrderSchema })
-  async validate(data: Record<string, unknown>): Promise<Record<string, unknown>> {
+  @Validate({ step: 2, name: 'validate', schema: OrderSchema })
+  async validate(data: Order): Promise<Order> {
     return data;
   }
 
   @Transform({ step: 3, name: 'enrich' })
-  async enrich(data: Record<string, unknown> & { items?: { qty: number; price: number }[] }): Promise<Record<string, unknown>> {
+  async enrich(data: Order): Promise<Order & { total: number; tax: number }> {
     const items = data.items ?? [];
     const subtotal = items.reduce((sum, i) => sum + i.qty * i.price, 0);
     const tax = subtotal * 0.1;
-    return {
-      ...data,
-      subtotal,
-      tax,
-      total: subtotal + tax,
-      processedAt: new Date().toISOString(),
-    };
+    return { ...data, subtotal, tax, total: subtotal + tax };
   }
 }
 ```
@@ -117,58 +119,159 @@ export class DataController {
 }
 ```
 
-## Batch processing with StreamService
+## Schema validation
 
-Process arrays through pipelines in batches:
+Build schemas with the fluent API. Full type inference via `Infer<T>`:
 
 ```typescript
-import { StreamService } from '@hazeljs/data';
+import { Schema, Infer, SchemaValidator } from '@hazeljs/data';
+
+const UserSchema = Schema.object({
+  email: Schema.string().email(),
+  name: Schema.string().min(1).max(200),
+  age: Schema.number().min(0).max(150),
+  role: Schema.string().oneOf(['user', 'admin', 'moderator', 'guest']),
+  active: Schema.boolean().default(true),
+});
 
-const streamService = new StreamService(etlService);
-const results = await streamService.processBatch(OrderProcessingPipeline, orders);
+type User = Infer<typeof UserSchema>;
+
+// Validate (throws on failure)
+const validator = new SchemaValidator();
+const user = validator.validate(UserSchema, rawData);
+
+// Safe validate (returns result)
+const result = validator.safeValidate(UserSchema, rawData);
+if (result.success) {
+  const user = result.data;
+} else {
+  console.error(result.errors);
+}
 ```
 
-## Schema validation
+### Schema types and modifiers
+
+| Type | Example |
+|------|---------|
+| `Schema.string()` | `.email()`, `.url()`, `.min()`, `.max()`, `.uuid()`, `.oneOf()`, `.pattern()`, `.required()`, `.trim()` |
+| `Schema.number()` | `.min()`, `.max()`, `.integer()`, `.positive()`, `.negative()`, `.multipleOf()` |
+| `Schema.boolean()` | `.default()` |
+| `Schema.date()` | `.min()`, `.max()`, `.default()` |
+| `Schema.object({...})` | `.strict()`, `.pick()`, `.omit()`, `.extend()` |
+| `Schema.array(itemSchema)` | `.min()`, `.max()`, `.nonempty()` |
+| `Schema.literal(value)` | Literal values |
+| `Schema.union([a, b])` | Discriminated unions |
+| Modifiers | `.optional()`, `.nullable()`, `.default()`, `.transform()`, `.refine()`, `.refineAsync()` |
 
-Build schemas with the fluent API:
+## Pipeline options
+
+Steps support conditional execution, retry, timeout, and DLQ:
 
 ```typescript
-import { Schema } from '@hazeljs/data';
+@Transform({
+  step: 2,
+  name: 'enrich',
+  when: (data) => (data as { type: string }).type === 'order',
+  retry: { attempts: 3, delay: 500, backoff: 'exponential' },
+  timeoutMs: 5000,
+  dlq: { handler: (item, err, step) => logger.error('DLQ', { item, err, step }) },
+})
+async enrich(data: unknown) {
+  return { ...data, enriched: true };
+}
+```
 
-const UserSchema = Schema.object()
-  .prop('email', Schema.string().format('email').required())
-  .prop('name', Schema.string().minLength(1).maxLength(200))
-  .prop('age', Schema.number().min(0).max(150))
-  .prop('role', Schema.string().oneOf(['user', 'admin', 'moderator', 'guest']))
-  .required();
+## PipelineBuilder (programmatic pipelines)
 
-const validator = new SchemaValidator();
-const { value, error } = validator.validate(UserSchema, rawData);
+Build pipelines in code without decorators:
+
+```typescript
+import { PipelineBuilder } from '@hazeljs/data';
+
+const pipeline = new PipelineBuilder('orders')
+  .addTransform('normalize', (d) => ({ ...d, email: (d as { email: string }).email?.toLowerCase() }))
+  .branch(
+    'classify',
+    (d) => (d as { type: string }).type === 'premium',
+    (b) => b.addTransform('enrichPremium', enrichPremium),
+    (b) => b.addTransform('enrichStandard', enrichStandard)
+  )
+  .parallel('enrich', [
+    (d) => ({ ...d, a: 1 }),
+    (d) => ({ ...d, b: 2 }),
+  ])
+  .catch((data, err) => ({ ...data, error: err.message }));
+
+const result = await pipeline.execute(rawData);
 ```
 
-## Data quality checks
+## Batch and stream processing
+
+```typescript
+import { StreamService, StreamProcessor } from '@hazeljs/data';
+
+// Batch
+const results = await streamService.processBatch(pipeline, items);
+
+// Streaming with windowing
+const processor = new StreamProcessor(etlService);
+for await (const batch of processor.tumblingWindow(source, 60_000)) {
+  console.log(batch.items, batch.windowStart, batch.windowEnd);
+}
+// Also: slidingWindow, sessionWindow, joinStreams
+```
+
+## Data quality
 
 ```typescript
 import { QualityService } from '@hazeljs/data';
 
 const qualityService = new QualityService();
-const report = await qualityService.check(records, {
-  completeness: ['id', 'email', 'createdAt'],
-  notNull: ['id', 'status'],
-});
+
+qualityService.registerCheck('completeness', qualityService.completeness(['id', 'email']));
+qualityService.registerCheck('notNull', qualityService.notNull(['id']));
+qualityService.registerCheck('uniqueness', qualityService.uniqueness(['id']));
+qualityService.registerCheck('range', qualityService.range('age', { min: 0, max: 120 }));
+qualityService.registerCheck('pattern', qualityService.pattern('phone', /^\d{10}$/));
+qualityService.registerCheck('ref', qualityService.referentialIntegrity('status', ['active', 'inactive']));
+
+const report = await qualityService.runChecks('users', records);
+const profile = qualityService.profile('users', records);
+const anomalies = qualityService.detectAnomalies(records, ['value'], 2);
 ```
 
-## Flink configuration (optional)
+## PII decorators
 
-For distributed stream processing with Apache Flink:
+```typescript
+import { Transform, Mask, Redact } from '@hazeljs/data';
+
+@Transform({ step: 1, name: 'sanitize' })
+@Mask({ fields: ['email', 'ssn'], showLast: 4 })
+sanitize(data: User) {
+  return data; // email/ssn already masked
+}
+
+@Transform({ step: 2, name: 'redact' })
+@Redact({ fields: ['internalId'] })
+redact(data: Record<string, unknown>) {
+  return data; // internalId removed
+}
+```
+
+## Test utilities
 
 ```typescript
-DataModule.forRoot({
-  flink: {
-    url: process.env.FLINK_REST_URL ?? 'http://localhost:8081',
-    timeout: 30000,
-  },
-});
+import { SchemaFaker, PipelineTestHarness, MockSource, MockSink } from '@hazeljs/data';
+
+const fake = SchemaFaker.generate(UserSchema);
+const many = SchemaFaker.generateMany(UserSchema, 10);
+
+const harness = PipelineTestHarness.create(etlService, pipeline);
+const { result, events } = await harness.run(input);
+await harness.runAndAssertSuccess(input);
+
+const source = new MockSource([{ x: 1 }]);
+const sink = new MockSink();
 ```
 
 ## Built-in transformers
@@ -182,9 +285,20 @@ DataModule.forRoot({
 | `omit` | Remove specific keys from objects |
 | `renameKeys` | Rename object keys |
 
+## Flink configuration (optional)
+
+```typescript
+DataModule.forRoot({
+  flink: {
+    url: process.env.FLINK_REST_URL ?? 'http://localhost:8081',
+    timeout: 30000,
+  },
+});
+```
+
 ## Example
 
-See [hazeljs-data-starter](../../../hazeljs-data-starter) for a full example with order and user pipelines, REST API, and quality reports.
+See [hazeljs-data-starter](../../../hazeljs-data-starter) for a full example with order and user pipelines, PipelineBuilder, PII decorators, quality profiling, anomaly detection, and REST API.
 
 ## Links
 

package/dist/connectors/connector.interface.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Connector interfaces for data sources and sinks.
+ * All connectors implement DataSource<T> or DataSink<T>.
+ */
+export interface DataSource<T = unknown> {
+    readonly name: string;
+    /** Open/initialize the connection */
+    open(): Promise<void>;
+    /** Close/release the connection */
+    close(): Promise<void>;
+    /** Read all records as an array */
+    readAll(): Promise<T[]>;
+    /** Read records as an async generator (streaming) */
+    read(): AsyncGenerator<T>;
+}
+export interface DataSink<T = unknown> {
+    readonly name: string;
+    open(): Promise<void>;
+    close(): Promise<void>;
+    /** Write a single record */
+    write(record: T): Promise<void>;
+    /** Write a batch of records (more efficient when supported) */
+    writeBatch(records: T[]): Promise<void>;
+}
+export interface ConnectorOptions {
+    /** Connector display name */
+    name?: string;
+}
+//# sourceMappingURL=connector.interface.d.ts.map

package/dist/connectors/connector.interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"connector.interface.d.ts","sourceRoot":"","sources":["../../src/connectors/connector.interface.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,OAAO;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,mCAAmC;IACnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,mCAAmC;IACnC,OAAO,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACxB,qDAAqD;IACrD,IAAI,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC;CAC3B;AAED,MAAM,WAAW,QAAQ,CAAC,CAAC,GAAG,OAAO;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,4BAA4B;IAC5B,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,+DAA+D;IAC/D,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,gBAAgB;IAC/B,6BAA6B;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf"}

package/dist/connectors/connector.interface.js

@@ -0,0 +1,6 @@
+"use strict";
+/**
+ * Connector interfaces for data sources and sinks.
+ * All connectors implement DataSource<T> or DataSink<T>.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/connectors/csv.connector.d.ts

@@ -0,0 +1,63 @@
+import type { DataSource, DataSink } from './connector.interface';
+export interface CsvSourceOptions {
+    filePath: string;
+    delimiter?: string;
+    hasHeader?: boolean;
+    /** Custom header names (used when hasHeader is false) */
+    headers?: string[];
+    name?: string;
+}
+export interface CsvSinkOptions {
+    filePath: string;
+    delimiter?: string;
+    /** Write header row on open */
+    writeHeader?: boolean;
+    name?: string;
+}
+/**
+ * CSV file data source — reads records from a CSV file.
+ *
+ * @example
+ * const source = new CsvSource({ filePath: './data.csv', hasHeader: true });
+ * await source.open();
+ * const records = await source.readAll();
+ * await source.close();
+ */
+export declare class CsvSource implements DataSource<Record<string, string>> {
+    readonly name: string;
+    private readonly filePath;
+    private readonly delimiter;
+    private readonly hasHeader;
+    private readonly customHeaders?;
+    constructor(options: CsvSourceOptions);
+    open(): Promise<void>;
+    close(): Promise<void>;
+    readAll(): Promise<Record<string, string>[]>;
+    read(): AsyncGenerator<Record<string, string>>;
+    private parseLine;
+}
+/**
+ * CSV file data sink — writes records to a CSV file.
+ *
+ * @example
+ * const sink = new CsvSink({ filePath: './output.csv', writeHeader: true });
+ * await sink.open();
+ * await sink.writeBatch(records);
+ * await sink.close();
+ */
+export declare class CsvSink implements DataSink<Record<string, unknown>> {
+    readonly name: string;
+    private readonly filePath;
+    private readonly delimiter;
+    private readonly writeHeader;
+    private headers;
+    private stream;
+    private headerWritten;
+    constructor(options: CsvSinkOptions);
+    open(): Promise<void>;
+    close(): Promise<void>;
+    write(record: Record<string, unknown>): Promise<void>;
+    writeBatch(records: Record<string, unknown>[]): Promise<void>;
+    private writeLine;
+}
+//# sourceMappingURL=csv.connector.d.ts.map

package/dist/connectors/csv.connector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.connector.d.ts","sourceRoot":"","sources":["../../src/connectors/csv.connector.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAElE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,qBAAa,SAAU,YAAW,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAW;gBAE9B,OAAO,EAAE,gBAAgB;IAQ/B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAMrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAQ3C,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA4BrD,OAAO,CAAC,SAAS;CAwBlB;AAED;;;;;;;;GAQG;AACH,qBAAa,OAAQ,YAAW,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,MAAM,CAAqD;IACnE,OAAO,CAAC,aAAa,CAAS;gBAElB,OAAO,EAAE,cAAc;IAO7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAUtB,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAerD,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAInE,OAAO,CAAC,SAAS;CAWlB"}

package/dist/connectors/csv.connector.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CsvSink = exports.CsvSource = void 0;
+const fs_1 = require("fs");
+const readline_1 = require("readline");
+/**
+ * CSV file data source — reads records from a CSV file.
+ *
+ * @example
+ * const source = new CsvSource({ filePath: './data.csv', hasHeader: true });
+ * await source.open();
+ * const records = await source.readAll();
+ * await source.close();
+ */
+class CsvSource {
+    constructor(options) {
+        this.name = options.name ?? `csv:${options.filePath}`;
+        this.filePath = options.filePath;
+        this.delimiter = options.delimiter ?? ',';
+        this.hasHeader = options.hasHeader ?? true;
+        this.customHeaders = options.headers;
+    }
+    async open() {
+        if (!(0, fs_1.existsSync)(this.filePath)) {
+            throw new Error(`CSV file not found: ${this.filePath}`);
+        }
+    }
+    async close() {
+        // No-op for file reads
+    }
+    async readAll() {
+        const records = [];
+        for await (const record of this.read()) {
+            records.push(record);
+        }
+        return records;
+    }
+    async *read() {
+        const rl = (0, readline_1.createInterface)({
+            input: (0, fs_1.createReadStream)(this.filePath),
+            crlfDelay: Infinity,
+        });
+        let headers = this.customHeaders ?? null;
+        let isFirst = true;
+        for await (const line of rl) {
+            const cols = this.parseLine(line);
+            if (isFirst && this.hasHeader && !this.customHeaders) {
+                headers = cols;
+                isFirst = false;
+                continue;
+            }
+            isFirst = false;
+            if (!headers) {
+                headers = cols.map((_, i) => `col${i}`);
+            }
+            const record = {};
+            headers.forEach((h, i) => {
+                record[h] = cols[i] ?? '';
+            });
+            yield record;
+        }
+    }
+    parseLine(line) {
+        const result = [];
+        let current = '';
+        let inQuote = false;
+        for (let i = 0; i < line.length; i++) {
+            const ch = line[i];
+            if (ch === '"') {
+                if (inQuote && line[i + 1] === '"') {
+                    current += '"';
+                    i++;
+                }
+                else {
+                    inQuote = !inQuote;
+                }
+            }
+            else if (ch === this.delimiter && !inQuote) {
+                result.push(current);
+                current = '';
+            }
+            else {
+                current += ch;
+            }
+        }
+        result.push(current);
+        return result;
+    }
+}
+exports.CsvSource = CsvSource;
+/**
+ * CSV file data sink — writes records to a CSV file.
+ *
+ * @example
+ * const sink = new CsvSink({ filePath: './output.csv', writeHeader: true });
+ * await sink.open();
+ * await sink.writeBatch(records);
+ * await sink.close();
+ */
+class CsvSink {
+    constructor(options) {
+        this.headers = null;
+        this.stream = null;
+        this.headerWritten = false;
+        this.name = options.name ?? `csv:${options.filePath}`;
+        this.filePath = options.filePath;
+        this.delimiter = options.delimiter ?? ',';
+        this.writeHeader = options.writeHeader ?? true;
+    }
+    async open() {
+        this.stream = (0, fs_1.createWriteStream)(this.filePath, { encoding: 'utf8' });
+    }
+    async close() {
+        return new Promise((resolve, reject) => {
+            if (!this.stream) {
+                resolve();
+                return;
+            }
+            this.stream.end((err) => (err ? reject(err) : resolve()));
+        });
+    }
+    async write(record) {
+        if (!this.stream)
+            throw new Error('CsvSink: call open() before write()');
+        if (!this.headers) {
+            this.headers = Object.keys(record);
+        }
+        if (this.writeHeader && !this.headerWritten) {
+            this.writeLine(this.headers);
+            this.headerWritten = true;
+        }
+        this.writeLine(this.headers.map((h) => String(record[h] ?? '')));
+    }
+    async writeBatch(records) {
+        for (const r of records)
+            await this.write(r);
+    }
+    writeLine(cols) {
+        const line = cols
+            .map((c) => c.includes(this.delimiter) || c.includes('"') || c.includes('\n')
+            ? `"${c.replace(/"/g, '""')}"`
+            : c)
+            .join(this.delimiter) + '\n';
+        this.stream.write(line);
+    }
+}
+exports.CsvSink = CsvSink;

package/dist/connectors/http.connector.d.ts

@@ -0,0 +1,68 @@
+import type { DataSource, DataSink } from './connector.interface';
+export interface HttpSourceOptions {
+    url: string;
+    method?: 'GET' | 'POST';
+    headers?: Record<string, string>;
+    body?: unknown;
+    /** JSON path to the array in the response (e.g. "data.items") */
+    dataPath?: string;
+    /** Pagination: next page URL extracted from response (e.g. "meta.next") */
+    nextPagePath?: string;
+    name?: string;
+    timeoutMs?: number;
+}
+export interface HttpSinkOptions {
+    url: string;
+    method?: 'POST' | 'PUT' | 'PATCH';
+    headers?: Record<string, string>;
+    /** Batch size — how many records to send per request (default: 1) */
+    batchSize?: number;
+    /** JSON field name to wrap records in (e.g. "records" → { records: [...] }) */
+    bodyKey?: string;
+    name?: string;
+    timeoutMs?: number;
+}
+/**
+ * HTTP API data source — reads records from a REST API.
+ * Supports pagination via `nextPagePath`.
+ *
+ * @example
+ * const source = new HttpSource({
+ *   url: 'https://api.example.com/users',
+ *   dataPath: 'data',
+ *   nextPagePath: 'meta.next',
+ * });
+ */
+export declare class HttpSource implements DataSource<unknown> {
+    readonly name: string;
+    private readonly options;
+    constructor(options: HttpSourceOptions);
+    open(): Promise<void>;
+    close(): Promise<void>;
+    readAll(): Promise<unknown[]>;
+    read(): AsyncGenerator<unknown>;
+    private fetchPage;
+}
+/**
+ * HTTP API data sink — writes records to a REST API endpoint.
+ *
+ * @example
+ * const sink = new HttpSink({
+ *   url: 'https://api.example.com/ingest',
+ *   method: 'POST',
+ *   batchSize: 100,
+ *   bodyKey: 'records',
+ * });
+ */
+export declare class HttpSink implements DataSink<unknown> {
+    readonly name: string;
+    private readonly options;
+    private buffer;
+    constructor(options: HttpSinkOptions);
+    open(): Promise<void>;
+    close(): Promise<void>;
+    write(record: unknown): Promise<void>;
+    writeBatch(records: unknown[]): Promise<void>;
+    private flush;
+}
+//# sourceMappingURL=http.connector.d.ts.map

package/dist/connectors/http.connector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.connector.d.ts","sourceRoot":"","sources":["../../src/connectors/http.connector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAElE,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD;;;;;;;;;;GAUG;AACH,qBAAa,UAAW,YAAW,UAAU,CAAC,OAAO,CAAC;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;gBAEhC,OAAO,EAAE,iBAAiB;IAKhC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAMrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAEtB,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAM5B,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC;IAwBtC,OAAO,CAAC,SAAS;CAYlB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,QAAS,YAAW,QAAQ,CAAC,OAAO,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,MAAM,CAAiB;gBAEnB,OAAO,EAAE,eAAe;IAK9B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAErB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOrC,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAKrC,KAAK;CA2BpB"}