@cdp-forge/plugin-pipeline-sdk 1.1.9 → 1.1.11

package/config/default.ts

@@ -0,0 +1,43 @@
+import { Config } from "@cdp-forge/types";
+type Parsed = string | number | boolean | string[] | number[];
+
+function env<T extends Parsed>(key: string, fallback: T, type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]'  = 'string'): T {
+    const val = process.env[key];
+    if (val === undefined) return fallback;
+    switch (type) {
+        case "string[]":
+            return val.split(',').map(s => s.trim()) as T;
+        case "number[]":
+            return val.split(',').map(s => Number(s.trim())) as T;
+        case "boolean":
+            return (val.toLowerCase() === 'true') as T;
+        case "number":
+            return Number(val) as T;
+        case "string":
+            return val as T;
+    }
+}
+
+const config: Config =  {
+    kafka: {
+        brokers: env<string[]>('KAFKA_BROKERS', ['cdp-forge-kafka-kafka-bootstrap:9092'], 'string[]'),
+        fistPipelineTopic: env("KAFKA_BASE_TOPIC", "logs")
+    },
+    mysql: {
+        uri: env("MYSQL_URI","mysql://root:cdp-forge-root-2024@cdp-forge-mysql:3306/cdpforge")
+    },
+    opensearch:{
+        url: env("OPENSEARCH_URL","https://opensearch-cluster-master:9200"),
+        username: env("OPENSEARCH_USERNAME","admin"),
+        password: env("OPENSEARCH_PASSWORD","CdpForge@2024!")
+    },
+    pipelinemanager: {
+        config_topic: env("PIPELINEMANAGER_CONFIG_TOPIC","config"),
+        url: env("PIPELINEMANAGER_URL","http://cdp-forge-core-pipeline-manager")
+    },
+    pod: {
+        name: env("CLIENT_ID", Math.random().toString(36).substring(2, 10))
+    }
+};
+
+export default config;

package/config/test.ts

@@ -0,0 +1 @@
+export default {};

package/dist/ConfigListener.d.ts

@@ -5,6 +5,7 @@ export default class ConfigListener {
     private consumer;
     private stage;
     private config;
+    private consumerReadyP;
     constructor(stage: PipelineSTage, config: Config);
     start(): Promise<void>;
 }

package/dist/ConfigListener.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ConfigListener.d.ts","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAiB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAIzD,MAAM,CAAC,OAAO,OAAO,cAAc;IAC/B,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAS;gBAEX,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;IAW1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAe/B"}
+{"version":3,"file":"ConfigListener.d.ts","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAiB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,CAAC,OAAO,OAAO,cAAc;IAC/B,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAgB;gBAE1B,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;IAc1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB/B"}

package/dist/ConfigListener.js

@@ -1,16 +1,22 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const kafkajs_1 = require("kafkajs");
-const podName = process.env.CLIENT_ID || Math.random().toString(36).substring(2, 10);
+const config_1 = __importDefault(require("config"));
 class ConfigListener {
     constructor(stage, config) {
         this.config = config;
         this.kafka = new kafkajs_1.Kafka({
-            clientId: config.plugin.name + `plugin-${podName}`,
+            clientId: config.plugin.name + `plugin-${config_1.default.get("pod.name")}`,
             brokers: config.kafkaConfig.brokers,
         });
-        this.consumer = this.kafka.consumer({ groupId: config.plugin.name + `plugin-${podName}` });
+        this.consumer = this.kafka.consumer({ groupId: config.plugin.name + `plugin-${config_1.default.get("pod.name")}` });
         this.stage = stage;
+        this.consumerReadyP = new Promise((resolve) => {
+            this.consumer.on("consumer.fetch_start", () => resolve());
+        });
     }
     async start() {
         await this.consumer.connect();
@@ -25,6 +31,7 @@ class ConfigListener {
                 await this.consumer.commitOffsets([{ topic, partition, offset: (BigInt(message.offset) + 1n).toString() }]);
             }
         });
+        await this.consumerReadyP;
     }
 }
 exports.default = ConfigListener;

package/dist/ConfigListener.js.map

@@ -1 +1 @@
-{"version":3,"file":"ConfigListener.js","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":";;AAAA,qCAA8D;AAI9D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAErF,MAAqB,cAAc;IAM/B,YAAY,KAAoB,EAAE,MAAc;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACnB,QAAQ,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE;YACnD,OAAO,EAAE,MAAM,CAAC,WAAY,CAAC,OAAO;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE,EAAC,CAAC,CAAC;QAE7F,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACpB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACtE,MAAM,MAAM,GAAkB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAE1F,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;oBACvD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpE,CAAC;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;YAChH,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AAhCD,iCAgCC"}
+{"version":3,"file":"ConfigListener.js","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":";;;;;AAAA,qCAA4D;AAG5D,oDAAmC;AAEnC,MAAqB,cAAc;IAO/B,YAAY,KAAoB,EAAE,MAAc;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACnB,QAAQ,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,gBAAa,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;YACzE,OAAO,EAAE,MAAM,CAAC,WAAY,CAAC,OAAO;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,gBAAa,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAC,CAAC,CAAC;QAEnH,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAChD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,GAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAClG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACpB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACtE,MAAM,MAAM,GAAkB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAE1F,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;oBACvD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpE,CAAC;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;YAChH,CAAC;SACJ,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,cAAc,CAAC;IAC9B,CAAC;CACJ;AArCD,iCAqCC"}

package/dist/PipelineStage.d.ts

@@ -9,6 +9,7 @@ export default class PipelineSTage {
     input: string | null;
     output: string | null;
     currentOperation: Promise<void>;
+    consumerReadyP: Promise<void>;
     constructor(plugin: PipelinePluginI, config: Config);
     start(inputTopic: string, outputTopic?: string | null): Promise<void>;
     private _start;

package/dist/PipelineStage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PipelineStage.d.ts","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAsB,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAO,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI/C,MAAM,CAAC,OAAO,OAAO,aAAa;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEpB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;IAa7C,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YASnE,MAAM;IA4Bd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAMb,KAAK;IAKb,OAAO,CAAC,UAAU,EAAE,MAAM,EAAC,WAAW,EAAE,MAAM,GAAC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;CAIjF"}
+{"version":3,"file":"PipelineStage.d.ts","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAsB,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAO,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG/C,MAAM,CAAC,OAAO,OAAO,aAAa;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAElB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;IAgB7C,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YASnE,MAAM;IA6Bd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAMb,KAAK;IAKb,OAAO,CAAC,UAAU,EAAE,MAAM,EAAC,WAAW,EAAE,MAAM,GAAC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;CAIjF"}

package/dist/PipelineStage.js

@@ -1,16 +1,22 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const kafkajs_1 = require("kafkajs");
-const podName = process.env.CLIENT_ID || 'default-client-id';
+const config_1 = __importDefault(require("config"));
 class PipelineSTage {
     constructor(plugin, config) {
         this.plugin = plugin;
         this.kafka = new kafkajs_1.Kafka({
-            clientId: config.plugin.name + `plugin-${podName}`,
+            clientId: config.plugin.name + `plugin-${config_1.default.get("pod.name")}`,
             brokers: config.kafkaConfig.brokers,
         });
         this.consumer = this.kafka.consumer({ groupId: config.plugin.name + `plugin` });
         this.producer = this.kafka.producer();
+        this.consumerReadyP = new Promise((resolve) => {
+            this.consumer.on("consumer.fetch_start", () => resolve());
+        });
         this.input = null;
         this.output = null;
         this.currentOperation = Promise.resolve();
@@ -49,6 +55,7 @@ class PipelineSTage {
                 }
             },
         });
+        await this.consumerReadyP;
     }
     async stop() {
         await this.currentOperation;

package/dist/PipelineStage.js.map

@@ -1 +1 @@
-{"version":3,"file":"PipelineStage.js","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":";;AAAA,qCAAwE;AAIxE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,mBAAmB,CAAC;AAE7D,MAAqB,aAAa;IAS9B,YAAY,MAAuB,EAAE,MAAc;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACrB,QAAQ,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE;YACnD,OAAO,EAAE,MAAM,CAAC,WAAY,CAAC,OAAO;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,cAA6B,IAAI;QAC7D,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,MAAM;QAChB,IAAG,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAE9C,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACxE,MAAM,GAAG,GAAQ,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC7E,IAAG,CAAC,GAAG;oBAAE,OAAO;gBAChB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACvD,IAAG,CAAC,aAAa,EAAE,CAAC;oBAClB,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;oBAC5G,OAAO;gBACT,CAAC;gBAED,IAAG,IAAI,CAAC,MAAM,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACvB,KAAK,EAAE,IAAI,CAAC,MAAM;wBAClB,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;qBACnD,CAAC,CAAC;oBACL,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC9G,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,UAAkB,EAAC,WAA6B;QAC1D,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;CACJ;AA1ED,gCA0EC"}
+{"version":3,"file":"PipelineStage.js","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":";;;;;AAAA,qCAAwE;AAGxE,oDAAmC;AAEnC,MAAqB,aAAa;IAU9B,YAAY,MAAuB,EAAE,MAAc;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACrB,QAAQ,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,UAAU,gBAAa,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;YACzE,OAAO,EAAE,MAAM,CAAC,WAAY,CAAC,OAAO;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAO,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAC5D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,cAA6B,IAAI;QAC7D,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,MAAM;QAChB,IAAG,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAE9C,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACxE,MAAM,GAAG,GAAQ,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC7E,IAAG,CAAC,GAAG;oBAAE,OAAO;gBAChB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACvD,IAAG,CAAC,aAAa,EAAE,CAAC;oBAClB,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;oBAC5G,OAAO;gBACT,CAAC;gBAED,IAAG,IAAI,CAAC,MAAM,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACvB,KAAK,EAAE,IAAI,CAAC,MAAM;wBAClB,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;qBACrD,CAAC,CAAC;oBACH,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC9G,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,cAAc,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,UAAkB,EAAC,WAA6B;QAC1D,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;CACJ;AA/ED,gCA+EC"}

package/dist/config/default.ts

@@ -0,0 +1,43 @@
+import { Config } from "@cdp-forge/types";
+type Parsed = string | number | boolean | string[] | number[];
+
+function env<T extends Parsed>(key: string, fallback: T, type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]'  = 'string'): T {
+    const val = process.env[key];
+    if (val === undefined) return fallback;
+    switch (type) {
+        case "string[]":
+            return val.split(',').map(s => s.trim()) as T;
+        case "number[]":
+            return val.split(',').map(s => Number(s.trim())) as T;
+        case "boolean":
+            return (val.toLowerCase() === 'true') as T;
+        case "number":
+            return Number(val) as T;
+        case "string":
+            return val as T;
+    }
+}
+
+const config: Config =  {
+    kafka: {
+        brokers: env<string[]>('KAFKA_BROKERS', ['cdp-forge-kafka-kafka-bootstrap:9092'], 'string[]'),
+        fistPipelineTopic: env("KAFKA_BASE_TOPIC", "logs")
+    },
+    mysql: {
+        uri: env("MYSQL_URI","mysql://root:cdp-forge-root-2024@cdp-forge-mysql:3306/cdpforge")
+    },
+    opensearch:{
+        url: env("OPENSEARCH_URL","https://opensearch-cluster-master:9200"),
+        username: env("OPENSEARCH_USERNAME","admin"),
+        password: env("OPENSEARCH_PASSWORD","CdpForge@2024!")
+    },
+    pipelinemanager: {
+        config_topic: env("PIPELINEMANAGER_CONFIG_TOPIC","config"),
+        url: env("PIPELINEMANAGER_URL","http://cdp-forge-core-pipeline-manager")
+    },
+    pod: {
+        name: env("CLIENT_ID", Math.random().toString(36).substring(2, 10))
+    }
+};
+
+export default config;

package/dist/config/test.ts

@@ -0,0 +1 @@
+export default {};

package/dist/index.d.ts

@@ -1,7 +1,9 @@
+import clusterConfig from 'config';
 export type { PipelinePluginI } from './plugin/PipelinePluginI';
 export { default as PipelineStage } from './PipelineStage';
 export { default as ConfigListener } from './ConfigListener';
-export { default as ConfigReader } from './ConfigReader';
+export { clusterConfig };
 export { start } from './startServer';
 export type * from '@cdp-forge/types';
+export { GoogleTopicsMap } from '@cdp-forge/types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,mBAAmB,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,aAAa,MAAM,QAAQ,CAAC;AACnC,YAAY,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,CAAC;AACzB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,mBAAmB,kBAAkB,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/index.js

@@ -3,13 +3,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.start = exports.ConfigReader = exports.ConfigListener = exports.PipelineStage = void 0;
+exports.GoogleTopicsMap = exports.start = exports.clusterConfig = exports.ConfigListener = exports.PipelineStage = void 0;
+const dotenv_1 = __importDefault(require("dotenv"));
+dotenv_1.default.config();
+const config_1 = __importDefault(require("config"));
+exports.clusterConfig = config_1.default;
 var PipelineStage_1 = require("./PipelineStage");
 Object.defineProperty(exports, "PipelineStage", { enumerable: true, get: function () { return __importDefault(PipelineStage_1).default; } });
 var ConfigListener_1 = require("./ConfigListener");
 Object.defineProperty(exports, "ConfigListener", { enumerable: true, get: function () { return __importDefault(ConfigListener_1).default; } });
-var ConfigReader_1 = require("./ConfigReader");
-Object.defineProperty(exports, "ConfigReader", { enumerable: true, get: function () { return __importDefault(ConfigReader_1).default; } });
 var startServer_1 = require("./startServer");
 Object.defineProperty(exports, "start", { enumerable: true, get: function () { return startServer_1.start; } });
+var types_1 = require("@cdp-forge/types");
+Object.defineProperty(exports, "GoogleTopicsMap", { enumerable: true, get: function () { return types_1.GoogleTopicsMap; } });
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;AACA,iDAA2D;AAAlD,+HAAA,OAAO,OAAiB;AACjC,mDAA6D;AAApD,iIAAA,OAAO,OAAkB;AAClC,+CAAyD;AAAhD,6HAAA,OAAO,OAAgB;AAChC,6CAAsC;AAA7B,oGAAA,KAAK,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,oDAAmC;AAI1B,wBAJF,gBAAa,CAIE;AAFtB,iDAA2D;AAAlD,+HAAA,OAAO,OAAiB;AACjC,mDAA6D;AAApD,iIAAA,OAAO,OAAkB;AAElC,6CAAsC;AAA7B,oGAAA,KAAK,OAAA;AAEd,0CAAkD;AAAzC,wGAAA,eAAe,OAAA"}

package/dist/startServer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"startServer.d.ts","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAE9C,wBAAsB,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;;;GA0BhE"}
+{"version":3,"file":"startServer.d.ts","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,cAAc,MAAM,kBAAkB,CAAC;AAE9C,wBAAsB,KAAK,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;;;GAyBhE"}

package/dist/startServer.js

@@ -9,19 +9,17 @@ const ConfigListener_1 = __importDefault(require("./ConfigListener"));
 async function start(plugin, config) {
     const stage = new PipelineStage_1.default(plugin, config);
     const configListener = new ConfigListener_1.default(stage, config);
-    configListener.start().then(() => {
-        fetch(`${config.manager.url}/register`, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-            },
-            body: JSON.stringify(config.plugin),
-        }).then(res => {
-            if (!res.ok) {
-                throw new Error(`Failed to register plugin with Template Manager: ${res.statusText}, request: ${JSON.stringify(config.plugin)}`);
-            }
-        });
+    await configListener.start();
+    const res = await fetch(`${config.manager.url}/register`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(config.plugin),
     });
+    if (!res.ok) {
+        throw new Error(`Failed to register plugin with Template Manager: ${res.statusText}, request: ${JSON.stringify(config.plugin)}`);
+    }
     const handleExit = async () => {
         console.log('Arresto del server in corso...');
         await stage.stop();

package/dist/startServer.js.map

@@ -1 +1 @@
-{"version":3,"file":"startServer.js","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":";;;;;AAKA,sBA0BG;AA7BH,oEAA4C;AAC5C,sEAA8C;AAEvC,KAAK,UAAU,KAAK,CAAC,MAAuB,EAAE,MAAc;IAC/D,MAAM,KAAK,GAAG,IAAI,uBAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,IAAI,wBAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,cAAc,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;QAC/B,KAAK,CAAC,GAAG,MAAM,CAAC,OAAQ,CAAC,GAAG,WAAW,EAAE;YACvC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YACZ,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,oDAAoD,GAAG,CAAC,UAAU,cAAc,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnI,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAElC,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;AACnC,CAAC"}
+{"version":3,"file":"startServer.js","sourceRoot":"","sources":["../src/startServer.ts"],"names":[],"mappings":";;;;;AAKA,sBAyBG;AA5BH,oEAA4C;AAC5C,sEAA8C;AAEvC,KAAK,UAAU,KAAK,CAAC,MAAuB,EAAE,MAAc;IAC/D,MAAM,KAAK,GAAG,IAAI,uBAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,IAAI,wBAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,MAAM,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,OAAQ,CAAC,GAAG,WAAW,EAAE;QACvD,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;KACtC,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,oDAAoD,GAAG,CAAC,UAAU,cAAc,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrI,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAElC,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;AACnC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@cdp-forge/plugin-pipeline-sdk",
-    "version": "1.1.9",
+    "version": "1.1.11",
     "description": "SDK for pipeline plugins for CDP Forge platform",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -34,17 +34,20 @@
     },
     "homepage": "https://github.com/CDPForge/plugin-pipeline-template#readme",
     "dependencies": {
-        "@cdp-forge/types": "^1.1.3",
+        "@cdp-forge/types": "^1.1.4",
+        "config": "^4.0.0",
+        "dotenv": "^17.2.0",
         "kafkajs": "^2.2.4",
-        "node-fetch": "^3.3.2",
-        "yaml": "^2.7.0"
+        "node-fetch": "^3.3.2"
     },
     "devDependencies": {
+        "@types/config": "^3.3.5",
         "@types/jest": "^29.5.14",
         "@types/node": "^20.12.12",
         "jest": "^29.7.0",
         "nodemon": "^3.1.0",
         "ts-jest": "^29.2.5",
+        "ts-node": "^10.9.2",
         "typescript": "^5.7.3"
     },
     "engines": {

package/config/config.yml

@@ -1,7 +0,0 @@
-# This file is generated by the helm package and should be mounted on the plugin container
-kafkaConfig:
-  brokers:
-  -  'localhost:36715'
-  config_topic: 'config'
-
-pluginManagerUrl: 'https://plugin_template_url'

package/config/plugin.yml

@@ -1,4 +0,0 @@
-plugin:
-  name: 'myPlugin'
-  priority: 1 # 1 to 100 (no need if parallel)
-  type: 'blocking' # or 'parallel'