@hazeljs/data 0.2.0-alpha.1

package/dist/streaming/flink/flink.client.js

@@ -0,0 +1,267 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlinkClient = void 0;
+/**
+ * Flink Client - REST API client for Apache Flink clusters
+ * Interacts with Flink JobManager REST API
+ */
+class FlinkClient {
+    constructor(config) {
+        this.config = {
+            timeout: 30000,
+            retries: 3,
+            ...config,
+        };
+        this.url = this.config.url.replace(/\/$/, '');
+    }
+    async request(method, path, body) {
+        const url = `${this.url}${path}`;
+        const headers = {
+            'Content-Type': 'application/json',
+        };
+        if (this.config.auth?.type === 'basic' && this.config.auth.username) {
+            const credentials = Buffer.from(`${this.config.auth.username}:${this.config.auth.password || ''}`).toString('base64');
+            headers['Authorization'] = `Basic ${credentials}`;
+        }
+        else if (this.config.auth?.type === 'token' && this.config.auth.token) {
+            headers['Authorization'] = `Bearer ${this.config.auth.token}`;
+        }
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+        try {
+            const response = await fetch(url, {
+                method,
+                headers,
+                body: body ? JSON.stringify(body) : undefined,
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                const text = await response.text();
+                throw new Error(`Flink API error ${response.status}: ${text}`);
+            }
+            const text = await response.text();
+            return text ? JSON.parse(text) : {};
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            if (error instanceof Error) {
+                throw new Error(`Flink request failed: ${error.message}`);
+            }
+            throw error;
+        }
+    }
+    async listJobs() {
+        const result = await this.request('GET', '/jobs/overview');
+        const jobs = result.jobs ?? [];
+        return jobs.map((j) => ({
+            id: j.id,
+            status: j.status,
+            startTime: j['start-time'],
+            endTime: j['end-time'],
+            duration: j.duration,
+        }));
+    }
+    async getJobStatus(jobId) {
+        const result = await this.request('GET', `/jobs/${jobId}`);
+        return {
+            state: result.state,
+            startTime: result['start-time'],
+            duration: result.duration,
+        };
+    }
+    async cancelJob(jobId) {
+        await this.request('PATCH', `/jobs/${jobId}?mode=cancel`);
+    }
+    async stopJob(jobId, savepointPath) {
+        const body = savepointPath ? { targetDirectory: savepointPath } : {};
+        return this.request('PATCH', `/jobs/${jobId}?mode=stop`, body);
+    }
+    async createSavepoint(jobId, savepointPath) {
+        const body = savepointPath ? { targetDirectory: savepointPath } : {};
+        return this.request('POST', `/jobs/${jobId}/savepoints`, body);
+    }
+    async getClusterInfo() {
+        return this.request('GET', '/overview');
+    }
+    async getTaskManagers() {
+        const result = await this.request('GET', '/taskmanagers');
+        return result.taskmanagers ?? [];
+    }
+    /**
+     * Upload a JAR file to the Flink cluster.
+     * @param jarPath Local filesystem path to the JAR file
+     * @returns The Flink JAR ID for subsequent run calls
+     */
+    async uploadJar(jarPath) {
+        const fs = await Promise.resolve().then(() => __importStar(require('fs')));
+        const path = await Promise.resolve().then(() => __importStar(require('path')));
+        if (!fs.existsSync(jarPath)) {
+            throw new Error(`JAR file not found: ${jarPath}`);
+        }
+        const url = `${this.url}/jars/upload`;
+        const fileName = path.basename(jarPath);
+        const fileBuffer = fs.readFileSync(jarPath);
+        const headers = {};
+        if (this.config.auth?.type === 'basic' && this.config.auth.username) {
+            headers['Authorization'] =
+                `Basic ${Buffer.from(`${this.config.auth.username}:${this.config.auth.password ?? ''}`).toString('base64')}`;
+        }
+        else if (this.config.auth?.type === 'token' && this.config.auth.token) {
+            headers['Authorization'] = `Bearer ${this.config.auth.token}`;
+        }
+        const boundary = `----HazelJSFormBoundary${Date.now()}`;
+        const bodyParts = [
+            Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="jarfile"; filename="${fileName}"\r\nContent-Type: application/java-archive\r\n\r\n`),
+            fileBuffer,
+            Buffer.from(`\r\n--${boundary}--\r\n`),
+        ];
+        const body = Buffer.concat(bodyParts);
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+        try {
+            const response = await fetch(url, {
+                method: 'POST',
+                headers: {
+                    ...headers,
+                    'Content-Type': `multipart/form-data; boundary=${boundary}`,
+                    'Content-Length': String(body.length),
+                },
+                body,
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                throw new Error(`Flink JAR upload failed ${response.status}: ${await response.text()}`);
+            }
+            const result = (await response.json());
+            const filename = result.filename ?? '';
+            const match = /\/jars\/([^/]+)$/.exec(filename);
+            if (!match)
+                throw new Error(`Could not extract JAR ID from Flink response: ${filename}`);
+            return match[1];
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            throw error instanceof Error ? error : new Error(String(error));
+        }
+    }
+    /**
+     * Run a previously uploaded JAR as a Flink job.
+     * @param jarId   The JAR ID returned by uploadJar()
+     * @param request Optional run parameters (entry class, parallelism, args, etc.)
+     * @returns The Flink job ID
+     */
+    async runJar(jarId, request = {}) {
+        const body = {};
+        if (request.jobName)
+            body['job-name'] = request.jobName;
+        if (request.parallelism)
+            body['parallelism'] = request.parallelism;
+        if (request.entryClass)
+            body['entry-class'] = request.entryClass;
+        if (request.programArgs)
+            body['program-args'] = request.programArgs;
+        if (request.savepointPath)
+            body['savepointPath'] = request.savepointPath;
+        if (request.allowNonRestoredState)
+            body['allowNonRestoredState'] = request.allowNonRestoredState;
+        const result = await this.request('POST', `/jars/${jarId}/run`, body);
+        if (!result.jobid)
+            throw new Error('Flink did not return a job ID');
+        return result.jobid;
+    }
+    /**
+     * Submit a SQL statement to the Flink SQL Gateway.
+     * Requires Flink SQL Gateway to be running.
+     * @param sql       The SQL statement to execute (CREATE TABLE, INSERT INTO, etc.)
+     * @param sessionId An existing Flink SQL Gateway session ID
+     */
+    async submitSql(sql, sessionId) {
+        const result = await this.request('POST', `/sessions/${sessionId}/statements`, { statement: sql });
+        const id = result.operationHandle?.identifier;
+        if (!id)
+            throw new Error('Flink SQL Gateway did not return an operation handle');
+        return id;
+    }
+    /**
+     * Create a new SQL Gateway session.
+     * @returns The session ID
+     */
+    async createSqlSession(properties) {
+        const result = await this.request('POST', '/sessions', { properties: properties ?? {} });
+        const id = result.sessionHandle?.identifier;
+        if (!id)
+            throw new Error('Flink SQL Gateway did not return a session handle');
+        return id;
+    }
+    /**
+     * Submit a job to Flink cluster.
+     * Requires `jarFile` path in the jobConfig or uses FlinkJobSubmitRequest.jarFile.
+     *
+     * Workflow: upload JAR → run JAR → return job ID.
+     */
+    async submitJob(jobConfig, request = {}) {
+        const jarFile = request.jarFile;
+        if (!jarFile) {
+            throw new Error('submitJob requires a JAR file path. Set request.jarFile to the path of your pipeline JAR, ' +
+                'or use submitSql() for Flink SQL-based pipelines.');
+        }
+        const jarId = await this.uploadJar(jarFile);
+        return this.runJar(jarId, {
+            jobName: request.jobName ?? jobConfig.jobName,
+            parallelism: request.parallelism ?? jobConfig.parallelism,
+            entryClass: request.entryClass,
+            programArgs: request.programArgs,
+            savepointPath: request.savepointPath,
+            allowNonRestoredState: request.allowNonRestoredState,
+        });
+    }
+    /**
+     * List uploaded JARs on the cluster.
+     */
+    async listJars() {
+        const result = await this.request('GET', '/jars');
+        return result.files ?? [];
+    }
+    /**
+     * Delete an uploaded JAR from the cluster.
+     */
+    async deleteJar(jarId) {
+        await this.request('DELETE', `/jars/${jarId}`);
+    }
+}
+exports.FlinkClient = FlinkClient;

package/dist/streaming/flink/flink.client.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flink.client.test.d.ts.map

package/dist/streaming/flink/flink.client.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flink.client.test.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.client.test.ts"],"names":[],"mappings":""}

package/dist/streaming/flink/flink.client.test.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const flink_client_1 = require("./flink.client");
+// Mock fetch for Flink REST API
+const mockFetch = jest.fn();
+describe('FlinkClient', () => {
+    let client;
+    beforeAll(() => {
+        global.fetch = mockFetch;
+    });
+    beforeEach(() => {
+        mockFetch.mockReset();
+        client = new flink_client_1.FlinkClient({ url: 'http://localhost:8081' });
+    });
+    it('listJobs fetches job overview', async () => {
+        mockFetch.mockResolvedValue({
+            ok: true,
+            text: () => Promise.resolve(JSON.stringify({
+                jobs: [
+                    { id: 'j1', status: 'RUNNING', 'start-time': 123, 'end-time': 456, duration: 100 },
+                ],
+            })),
+        });
+        const jobs = await client.listJobs();
+        expect(jobs).toHaveLength(1);
+        expect(jobs[0].id).toBe('j1');
+        expect(jobs[0].status).toBe('RUNNING');
+        expect(mockFetch).toHaveBeenCalledWith('http://localhost:8081/jobs/overview', expect.any(Object));
+    });
+    it('getJobStatus fetches job details', async () => {
+        mockFetch.mockResolvedValue({
+            ok: true,
+            text: () => Promise.resolve(JSON.stringify({ state: 'RUNNING', 'start-time': 123, duration: 500 })),
+        });
+        const status = await client.getJobStatus('j1');
+        expect(status.state).toBe('RUNNING');
+        expect(status.duration).toBe(500);
+    });
+    it('throws on API error', async () => {
+        mockFetch.mockResolvedValue({ ok: false, text: () => Promise.resolve('Not found') });
+        await expect(client.getJobStatus('bad')).rejects.toThrow('Flink API error');
+    });
+    it('submitJob requires a jarFile path', async () => {
+        await expect(client.submitJob({})).rejects.toThrow('submitJob requires a JAR file path');
+    });
+    it('getClusterInfo', async () => {
+        mockFetch.mockResolvedValue({
+            ok: true,
+            text: () => Promise.resolve(JSON.stringify({ taskmanagers: 2, 'slots-total': 8 })),
+        });
+        const info = await client.getClusterInfo();
+        expect(info.taskmanagers).toBe(2);
+    });
+    it('cancelJob', async () => {
+        mockFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('') });
+        await client.cancelJob('j1');
+        expect(mockFetch).toHaveBeenCalledWith(expect.stringContaining('mode=cancel'), expect.any(Object));
+    });
+});

package/dist/streaming/flink/flink.job.d.ts

@@ -0,0 +1,29 @@
+import { FlinkClient } from './flink.client';
+import type { FlinkJobConfig } from '../../data.types';
+import type { FlinkJobSubmitRequest } from './flink.client';
+export interface FlinkJobResult {
+    jobId: string;
+    status: string;
+    webUI?: string;
+}
+/**
+ * Flink Job - Job management operations
+ */
+export declare class FlinkJob {
+    private readonly client;
+    constructor(client: FlinkClient);
+    getStatus(jobId: string): Promise<{
+        state: string;
+        startTime?: number;
+        duration?: number;
+    }>;
+    cancel(jobId: string): Promise<void>;
+    createSavepoint(jobId: string, savepointPath?: string): Promise<{
+        'request-id': string;
+    }>;
+    stop(jobId: string, savepointPath?: string): Promise<{
+        'request-id': string;
+    }>;
+    submit(config: FlinkJobConfig, request?: FlinkJobSubmitRequest): Promise<string>;
+}
+//# sourceMappingURL=flink.job.d.ts.map

package/dist/streaming/flink/flink.job.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flink.job.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE5D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,qBAAa,QAAQ;IACP,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,WAAW;IAE1C,SAAS,CACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9D,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAIzF,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAI9E,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC;CAGvF"}

package/dist/streaming/flink/flink.job.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FlinkJob = void 0;
+/**
+ * Flink Job - Job management operations
+ */
+class FlinkJob {
+    constructor(client) {
+        this.client = client;
+    }
+    async getStatus(jobId) {
+        return this.client.getJobStatus(jobId);
+    }
+    async cancel(jobId) {
+        return this.client.cancelJob(jobId);
+    }
+    async createSavepoint(jobId, savepointPath) {
+        return this.client.createSavepoint(jobId, savepointPath);
+    }
+    async stop(jobId, savepointPath) {
+        return this.client.stopJob(jobId, savepointPath);
+    }
+    async submit(config, request) {
+        return this.client.submitJob(config, request);
+    }
+}
+exports.FlinkJob = FlinkJob;

package/dist/streaming/flink/flink.job.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flink.job.test.d.ts.map

package/dist/streaming/flink/flink.job.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flink.job.test.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.job.test.ts"],"names":[],"mappings":""}

package/dist/streaming/flink/flink.job.test.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const flink_client_1 = require("./flink.client");
+const flink_job_1 = require("./flink.job");
+const mockFetch = jest.fn();
+describe('FlinkJob', () => {
+    let job;
+    beforeAll(() => {
+        global.fetch = mockFetch;
+    });
+    beforeEach(() => {
+        mockFetch.mockReset();
+        const client = new flink_client_1.FlinkClient({ url: 'http://localhost:8081' });
+        job = new flink_job_1.FlinkJob(client);
+    });
+    it('getStatus fetches job status', async () => {
+        mockFetch.mockResolvedValue({
+            ok: true,
+            text: () => Promise.resolve(JSON.stringify({ state: 'RUNNING', duration: 100 })),
+        });
+        const status = await job.getStatus('j1');
+        expect(status.state).toBe('RUNNING');
+    });
+    it('cancel calls cancel endpoint', async () => {
+        mockFetch.mockResolvedValue({ ok: true, text: () => Promise.resolve('') });
+        await job.cancel('j1');
+        expect(mockFetch).toHaveBeenCalledWith(expect.stringContaining('/jobs/j1'), expect.any(Object));
+    });
+    it('createSavepoint', async () => {
+        mockFetch.mockResolvedValue({
+            ok: true,
+            text: () => Promise.resolve(JSON.stringify({ 'request-id': 'req-1' })),
+        });
+        const result = await job.createSavepoint('j1', '/path');
+        expect(result['request-id']).toBe('req-1');
+    });
+});

package/dist/streaming/flink/flink.operators.d.ts

@@ -0,0 +1,35 @@
+import type { PipelineStep } from '../../pipelines/etl.service';
+export type FlinkOperatorType = 'map' | 'filter' | 'flatMap' | 'window' | 'keyBy';
+export interface FlinkOperator {
+    type: FlinkOperatorType;
+    step: number;
+    name: string;
+    function?: (value: unknown) => unknown | Promise<unknown>;
+    predicate?: (value: unknown) => boolean;
+    windowType?: 'tumbling' | 'sliding' | 'session';
+    windowSize?: string;
+    aggregator?: (value: unknown) => unknown;
+}
+/**
+ * Maps HazelJS pipeline steps to Flink operators
+ * Note: Actual function/predicate execution happens in deployed Flink job
+ */
+export declare function mapToFlinkOperator(step: PipelineStep): FlinkOperator;
+export declare function createFlinkJobGraph(steps: PipelineStep[], sourceConfig: {
+    type: string;
+    topic?: string;
+    properties?: Record<string, string>;
+}, sinkConfig: {
+    type: string;
+    topic?: string;
+    properties?: Record<string, string>;
+}): {
+    source: typeof sourceConfig;
+    transformations: Array<{
+        step: number;
+        name: string;
+        operator: FlinkOperator;
+    }>;
+    sink: typeof sinkConfig;
+};
+//# sourceMappingURL=flink.operators.d.ts.map

package/dist/streaming/flink/flink.operators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flink.operators.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.operators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;AAElF,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;IACxC,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;IAChD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;CAC1C;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,CAuBpE;AAED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,YAAY,EAAE,EACrB,YAAY,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,EACnF,UAAU,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAChF;IACD,MAAM,EAAE,OAAO,YAAY,CAAC;IAC5B,eAAe,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAChF,IAAI,EAAE,OAAO,UAAU,CAAC;CACzB,CAUA"}

package/dist/streaming/flink/flink.operators.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapToFlinkOperator = mapToFlinkOperator;
+exports.createFlinkJobGraph = createFlinkJobGraph;
+/**
+ * Maps HazelJS pipeline steps to Flink operators
+ * Note: Actual function/predicate execution happens in deployed Flink job
+ */
+function mapToFlinkOperator(step) {
+    switch (step.type) {
+        case 'transform':
+            return {
+                type: 'map',
+                step: step.step,
+                name: step.name,
+                // Method name for job graph - actual handler in deployed job
+            };
+        case 'validate':
+            return {
+                type: 'filter',
+                step: step.step,
+                name: step.name,
+                predicate: () => true,
+            };
+        default:
+            return {
+                type: 'map',
+                step: step.step,
+                name: step.name,
+            };
+    }
+}
+function createFlinkJobGraph(steps, sourceConfig, sinkConfig) {
+    return {
+        source: sourceConfig,
+        transformations: steps.map((step) => ({
+            step: step.step,
+            name: step.name,
+            operator: mapToFlinkOperator(step),
+        })),
+        sink: sinkConfig,
+    };
+}

package/dist/streaming/flink/flink.operators.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=flink.operators.test.d.ts.map

package/dist/streaming/flink/flink.operators.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flink.operators.test.d.ts","sourceRoot":"","sources":["../../../src/streaming/flink/flink.operators.test.ts"],"names":[],"mappings":""}

package/dist/streaming/flink/flink.operators.test.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const flink_operators_1 = require("./flink.operators");
+describe('flink operators', () => {
+    describe('mapToFlinkOperator', () => {
+        it('maps transform step to map operator', () => {
+            const step = {
+                step: 1,
+                name: 'parse',
+                type: 'transform',
+                method: 'parse',
+            };
+            const op = (0, flink_operators_1.mapToFlinkOperator)(step);
+            expect(op.type).toBe('map');
+            expect(op.step).toBe(1);
+            expect(op.name).toBe('parse');
+        });
+        it('maps validate step to filter operator', () => {
+            const step = {
+                step: 2,
+                name: 'validate',
+                type: 'validate',
+                method: 'validate',
+            };
+            const op = (0, flink_operators_1.mapToFlinkOperator)(step);
+            expect(op.type).toBe('filter');
+        });
+    });
+    describe('createFlinkJobGraph', () => {
+        it('creates job graph with source, sink, transformations', () => {
+            const steps = [{ step: 1, name: 't1', type: 'transform', method: 't1' }];
+            const graph = (0, flink_operators_1.createFlinkJobGraph)(steps, { type: 'kafka', topic: 'in', properties: {} }, { type: 'kafka', topic: 'out', properties: {} });
+            expect(graph.source.topic).toBe('in');
+            expect(graph.sink.topic).toBe('out');
+            expect(graph.transformations).toHaveLength(1);
+        });
+    });
+});

package/dist/streaming/stream.builder.d.ts

@@ -0,0 +1,22 @@
+import { ETLService } from '../pipelines/etl.service';
+import type { FlinkJobConfig } from '../data.types';
+import type { PipelineStep } from '../pipelines/etl.service';
+import { createFlinkJobGraph } from './flink/flink.operators';
+export interface StreamPipelineConfig {
+    source: string;
+    sink: string;
+    parallelism?: number;
+    steps: PipelineStep[];
+}
+/**
+ * Stream Builder - Builds Flink job config from @Stream decorated pipeline
+ */
+export declare class StreamBuilder {
+    private readonly etlService;
+    constructor(etlService: ETLService);
+    buildConfig(pipelineInstance: object, overrideConfig?: Partial<FlinkJobConfig>): {
+        jobConfig: FlinkJobConfig;
+        jobGraph: ReturnType<typeof createFlinkJobGraph>;
+    };
+}
+//# sourceMappingURL=stream.builder.d.ts.map

package/dist/streaming/stream.builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.builder.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAE9D,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,qBAAa,aAAa;IACZ,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAAV,UAAU,EAAE,UAAU;IAEnD,WAAW,CACT,gBAAgB,EAAE,MAAM,EACxB,cAAc,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GACvC;QACD,SAAS,EAAE,cAAc,CAAC;QAC1B,QAAQ,EAAE,UAAU,CAAC,OAAO,mBAAmB,CAAC,CAAC;KAClD;CA6CF"}

package/dist/streaming/stream.builder.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamBuilder = void 0;
+const decorators_1 = require("../decorators");
+const flink_operators_1 = require("./flink/flink.operators");
+/**
+ * Stream Builder - Builds Flink job config from @Stream decorated pipeline
+ */
+class StreamBuilder {
+    constructor(etlService) {
+        this.etlService = etlService;
+    }
+    buildConfig(pipelineInstance, overrideConfig) {
+        const metadata = (0, decorators_1.getStreamMetadata)(pipelineInstance.constructor);
+        if (!metadata) {
+            throw new Error('Pipeline must be decorated with @Stream');
+        }
+        const steps = this.etlService.extractSteps(pipelineInstance);
+        const sourceTopic = metadata.source.replace('kafka://', '');
+        const sinkTopic = metadata.sink.replace('kafka://', '');
+        const jobConfig = {
+            jobName: pipelineInstance.constructor.name,
+            parallelism: metadata.parallelism ?? 4,
+            checkpointInterval: 60000,
+            restartStrategy: {
+                type: 'fixed-delay',
+                attempts: 3,
+                delay: 10000,
+            },
+            ...overrideConfig,
+        };
+        const jobGraph = (0, flink_operators_1.createFlinkJobGraph)(steps, {
+            type: 'kafka',
+            topic: sourceTopic,
+            properties: {
+                'bootstrap.servers': process.env.KAFKA_BOOTSTRAP_SERVERS ?? 'localhost:9092',
+                'group.id': `hazeljs-${metadata.name}`,
+                'auto.offset.reset': 'latest',
+            },
+        }, {
+            type: 'kafka',
+            topic: sinkTopic,
+            properties: {
+                'bootstrap.servers': process.env.KAFKA_BOOTSTRAP_SERVERS ?? 'localhost:9092',
+            },
+        });
+        return { jobConfig, jobGraph };
+    }
+}
+exports.StreamBuilder = StreamBuilder;

package/dist/streaming/stream.builder.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=stream.builder.test.d.ts.map

package/dist/streaming/stream.builder.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.builder.test.d.ts","sourceRoot":"","sources":["../../src/streaming/stream.builder.test.ts"],"names":[],"mappings":""}