@cdp-forge/plugin-pipeline-sdk 1.1.4

package/README.md

@@ -0,0 +1,352 @@
+# CDP Forge Plugin Pipeline SDK
+
+SDK for easily implementing pipeline plugins for the CDP Forge platform.
+
+This project serves as an SDK for building plugins that can be integrated into the data processing pipeline of the CDP Forge platform. It is designed to simplify the development of custom data transformation and processing logic within the platform ecosystem.
+
+## 📦 Installation as NPM Library
+
+You can install this library as a dependency in other projects:
+
+```bash
+npm install plugin-pipeline-sdk
+```
+
+### Usage as Library
+
+```typescript
+import { 
+    PipelinePluginI, 
+    PipelineStage, 
+    ConfigListener, 
+    ConfigReader,
+    Log,
+    start
+} from 'plugin-pipeline-sdk';
+
+// Create a custom plugin
+class MyCustomPlugin implements PipelinePluginI {
+    async elaborate(log: Log): Promise<Log | null> {
+        // Implement your processing logic
+        console.log('Processing log:', log);
+        return log;
+    }
+
+    async init(): Promise<void> {
+        console.log('Plugin initialization');
+    }
+}
+
+// Load configuration
+const config = ConfigReader.getInstance('./config/config.yml', './config/plugin.yml').config;
+
+// Create plugin instance and start the server
+const customPlugin = new MyCustomPlugin();
+start(customPlugin, config).then(({ stage, configListener }) => {
+    console.log('Server started successfully');
+}).catch(error => {
+    console.error('Error during startup:', error);
+});
+```
+
+## 🚀 Features
+
+- **Pipeline Plugin:** Provides a structure for creating plugins that fit into a sequential or parallel processing pipeline
+- **Kafka Integration:** Uses Kafka for asynchronous communication and data streaming between pipeline stages
+- **TypeScript:** Written in TypeScript to improve code maintainability, type safety, and developer productivity
+- **Docker Support:** Includes Docker configuration for deployment
+- **Testing:** Jest configuration for unit tests
+- **Configuration Management:** Automatic merging of cluster and plugin configurations
+
+## 📋 Prerequisites
+
+- Node.js 20.11.1 or higher
+- npm or yarn
+- Docker (optional, for deployment)
+- Access to a Kafka cluster
+
+## 🛠️ Installation
+
+1. **Clone the repository:**
+   ```bash
+   git clone <repository-url>
+   cd plugin-pipeline-template
+   ```
+
+2. **Install dependencies:**
+   ```bash
+   npm install
+   ```
+
+3. **Configure the environment:**
+   - Copy and modify configuration files in `config/`
+   - Ensure Kafka brokers are accessible
+
+## ⚙️ Configuration
+
+The SDK uses two separate configuration files to manage different aspects of the plugin system:
+
+### Configuration File Structure
+
+#### `config/config.yml` - Cluster Configuration
+This file contains the **cluster-level configuration** that is **shared across all plugins** in the CDP Forge platform.
+```yaml
+kafkaConfig:
+  brokers:
+    - 'localhost:36715'
+
+manager: 
+  url: 'https://plugin_template_url'
+  config_topic: 'config'
+
+mysql:
+  uri: 'mysql://user:password@my-server-ip:3306'
+```
+
+**Important**: If you're using the **Helm installer** provided by the CDP Forge platform, this file is **automatically generated** and you don't need to modify it manually.
+
+#### `config/plugin.yml` - Plugin-Specific Configuration
+This file contains **plugin-specific settings** that define how your individual plugin behaves within the pipeline.
+
+```yaml
+plugin:
+  name: 'myPlugin'
+  priority: 1 # 1 to 100 (not required if parallel)
+  type: 'blocking' # or 'parallel'
+
+[any other useful string for the plugin]
+```
+
+### Field Descriptions
+
+#### Cluster Configuration (`config.yml`)
+- **`kafkaConfig.brokers`**  
+  List of Kafka broker addresses to which the plugin will connect. This is configured at the cluster level and shared by all plugins.
+
+- **`manager.url`**  
+  URL used to register or communicate with the plugin manager service.
+
+- **`manager.config_topic`**  
+  Kafka topic used for plugin configuration management across the cluster.
+
+- **`mysql.uri`**  
+  MySQL connection string for database operations.
+
+#### Plugin Configuration (`plugin.yml`)
+- **`plugin.name`**  
+  Unique identifier for your plugin instance within the pipeline.
+
+- **`plugin.priority`**  
+  (Required only for `blocking` plugins)  
+  An integer from **1 to 100** that defines the execution order of the plugin within the pipeline. A lower number means higher priority, so the plugin with priority 1 will be executed before plugins with priority 2,3,4...
+
+- **`plugin.type`**  
+  Defines the plugin execution mode:  
+  - `blocking`: The plugin processes data and returns a `Promise<Log>` for the next stage.  
+  - `parallel`: The plugin runs independently and returns a `Promise<void>`.
+
+### Configuration Management
+
+- **Cluster Config (`config.yml`)**: Managed by the platform, automatically generated by Helm installer
+- **Plugin Config (`plugin.yml`)**: Managed by you, defines your plugin's behavior
+- **Environment Variables**: Can override both configurations if needed
+- **Runtime Updates**: Plugin configuration can be updated without restarting the cluster
+
+### Using ConfigReader for Convenience
+
+The SDK provides a `ConfigReader` utility that automatically merges both configuration files into a single `config` object, making it easier to access all settings in your plugin code.
+
+```typescript
+import { ConfigReader } from 'plugin-pipeline-sdk';
+
+// The ConfigReader automatically loads and merges:
+// - config/config.yml (cluster configuration)
+// - config/plugin.yml (plugin configuration)
+const config = ConfigReader.getInstance('./config/config.yml', './config/plugin.yml').config;
+
+// Access cluster configuration
+console.log(config.kafkaConfig.brokers);
+console.log(config.manager.url);
+
+// Access plugin configuration
+console.log(config.plugin.name);
+console.log(config.plugin.priority);
+
+// Access merged configuration
+console.log(config.mysql.uri);
+```
+
+### Starting the Server with Configuration
+
+The `start()` function requires the merged configuration to initialize the server:
+
+```typescript
+import { start, PipelinePluginI, Log, ConfigReader } from 'plugin-pipeline-sdk';
+
+const config = ConfigReader.getInstance('./config/config.yml', './config/plugin.yml').config;
+
+class MyPlugin implements PipelinePluginI {
+    async elaborate(log: Log): Promise<Log | null> {
+        // Your plugin logic here
+        return log;
+    }
+
+    async init(): Promise<void> {
+        // Plugin initialization
+    }
+}
+
+// Start the server with the merged configuration
+start(new MyPlugin(), config).then(({ stage, configListener }) => {
+    console.log('Server started with merged configuration');
+}).catch(error => {
+    console.error('Error starting server:', error);
+});
+```
+
+The server will:
+1. **Load** both configuration files using the specified paths
+2. **Merge** them into a single config object
+3. **Validate** the configuration
+4. **Start** the plugin with the merged settings
+
+## 🔧 Plugin Development
+
+To create a new plugin, follow these steps:
+
+1. **Configure the `config.yml` and `plugin.yml` files correctly**
+2. **Implement the `elaborate` function in your plugin class**
+
+### Plugin Implementation
+
+The plugin must implement the `PipelinePluginI` interface:
+
+```typescript
+import { PipelinePluginI, Log } from 'plugin-pipeline-sdk';
+
+export default class MyPlugin implements PipelinePluginI {
+    elaborate(log: Log): Promise<Log | null> {
+        // Implement your processing logic here
+        // For blocking plugins: return Promise<Log>
+        // For parallel plugins: return Promise<void>
+        return Promise.resolve(log);
+    }
+
+    init(): Promise<void> {
+        // Plugin initialization
+        return Promise.resolve();
+    }
+}
+```
+
+### Plugin Types
+
+Depending on the plugin type:
+- **`blocking` plugins**: The `elaborate` function must return a `Promise<Log>`.
+- **`parallel` plugins**: The `elaborate` function must return a `Promise<void>`.
+
+## 📁 Project Structure
+
+```
+plugin-pipeline-template/
+├── config/                 # Configuration files
+│   ├── config.yml         # Cluster configuration
+│   └── plugin.yml         # Plugin-specific configuration
+├── src/                   # TypeScript source code
+│   ├── plugin/           # Plugin implementation
+│   │   ├── Plugin.ts     # Main plugin class
+│   │   └── PipelinePluginI.ts # Plugin interface
+│   ├── types.ts          # Type definitions
+│   ├── config.ts         # Configuration management
+│   ├── index.ts          # Library entry point
+│   └── ...               # Other utility files
+├── __tests__/            # Unit tests
+├── Dockerfile            # Docker configuration
+├── package.json          # Dependencies and scripts
+└── tsconfig.json         # TypeScript configuration
+```
+
+## 🚀 Available Scripts
+
+- **`npm run build`**: Compiles TypeScript code
+- **`npm test`**: Runs unit tests
+- **`npm run clean`**: Cleans the dist folder
+- **`npm run prepublishOnly`**: Builds before publishing
+
+## 🐳 Docker Deployment
+
+1. **Build the image:**
+   ```bash
+   docker build -t plugin-pipeline-sdk .
+   ```
+
+2. **Run the container:**
+   ```bash
+   docker run -p 3000:3000 plugin-pipeline-sdk
+   ```
+
+## 📊 Data Structure
+
+The plugin processes `Log` objects that contain:
+
+```typescript
+interface Log {
+    client: number;
+    date: string;
+    device: {
+        browser?: string;
+        id: string;
+        ip?: string;
+        os?: string;
+        type?: string;
+        userAgent?: string;
+    };
+    event: string;
+    geo?: {
+        city?: string;
+        country?: string;
+        point?: {
+            type: string;
+            coordinates: number[];
+        };
+        region?: string;
+    };
+    googleTopics?: GoogleTopic[];
+    instance: number;
+    page: {
+        description?: string;
+        href?: string;
+        image?: string;
+        title: string;
+        type?: string;
+    };
+    product?: Product[];
+    referrer?: string;
+    session: string;
+    target?: string;
+    order?: string;
+    [key: string]: any; // Allows additional properties
+}
+```
+
+## 📦 Publishing to NPM
+
+To publish this library to npm, see the [Publishing Guide](PUBLISHING.md).
+
+## 🤝 Contributing
+
+Contributions are welcome! To contribute:
+
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/AmazingFeature`)
+3. Commit your changes (`git commit -m 'Add some AmazingFeature'`)
+4. Push to the branch (`git push origin feature/AmazingFeature`)
+5. Open a Pull Request
+
+## 📄 License
+
+This project is distributed under the GPL-3.0 license. See the `LICENSE` file for more details.
+
+## 📞 Support
+
+For support and questions, please open an issue on the GitHub repository.

package/config/config.yml

@@ -0,0 +1,7 @@
+# This file is generated by the helm package and should be mounted on the plugin container
+kafkaConfig:
+  brokers:
+  -  'localhost:36715'
+  config_topic: 'config'
+
+pluginManagerUrl: 'https://plugin_template_url'

package/config/plugin.yml

@@ -0,0 +1,4 @@
+plugin:
+  name: 'myPlugin'
+  priority: 1 # 1 to 100 (no need if parallel)
+  type: 'blocking' # or 'parallel'

package/dist/ConfigListener.d.ts

@@ -0,0 +1,11 @@
+import PipelineSTage from "./PipelineStage";
+import { Config } from "@cdp-forge/types";
+export default class ConfigListener {
+    private kafka;
+    private consumer;
+    private stage;
+    private config;
+    constructor(stage: PipelineSTage, config: Config);
+    start(): Promise<void>;
+}
+//# sourceMappingURL=ConfigListener.d.ts.map

package/dist/ConfigListener.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConfigListener.d.ts","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":"AACA,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAiB,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAIzD,MAAM,CAAC,OAAO,OAAO,cAAc;IAC/B,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,MAAM,CAAS;gBAEX,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;IAW1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAe/B"}

package/dist/ConfigListener.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkajs_1 = require("kafkajs");
+const podName = process.env.CLIENT_ID || Math.random().toString(36).substring(2, 10);
+class ConfigListener {
+    constructor(stage, config) {
+        this.config = config;
+        this.kafka = new kafkajs_1.Kafka({
+            clientId: config.plugin.name + `plugin-${podName}`,
+            brokers: config.kafkaConfig.brokers,
+        });
+        this.consumer = this.kafka.consumer({ groupId: config.plugin.name + `plugin-${podName}` });
+        this.stage = stage;
+    }
+    async start() {
+        await this.consumer.connect();
+        await this.consumer.subscribe({ topic: this.config.manager.config_topic, fromBeginning: false });
+        await this.consumer.run({
+            autoCommit: false,
+            eachMessage: async ({ topic, partition, message }) => {
+                const configs = message.value ? JSON.parse(message.value.toString()) : null;
+                const config = configs.find(config => config.plugin === this.config.plugin.name);
+                if (config) {
+                    await this.stage.restart(config.inputTopic, config.outputTopic);
+                }
+                await this.consumer.commitOffsets([{ topic, partition, offset: (BigInt(message.offset) + 1n).toString() }]);
+            }
+        });
+    }
+}
+exports.default = ConfigListener;
+//# sourceMappingURL=ConfigListener.js.map

package/dist/ConfigListener.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConfigListener.js","sourceRoot":"","sources":["../src/ConfigListener.ts"],"names":[],"mappings":";;AAAA,qCAA8D;AAI9D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAErF,MAAqB,cAAc;IAM/B,YAAY,KAAoB,EAAE,MAAc;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACnB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE;YAClD,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE,EAAC,CAAC,CAAC;QAE5F,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACpB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACtE,MAAM,OAAO,GAAoB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC7F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjF,IAAG,MAAM,EAAE,CAAC;oBACR,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpE,CAAC;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;YAChH,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AAhCD,iCAgCC"}

package/dist/ConfigReader.d.ts

@@ -0,0 +1,6 @@
+import { Config } from '@cdp-forge/types';
+export default class ConfigReader {
+    static config: Config;
+    static generate(configPath: string, pluginPath: string): Config;
+}
+//# sourceMappingURL=ConfigReader.d.ts.map

package/dist/ConfigReader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConfigReader.d.ts","sourceRoot":"","sources":["../src/ConfigReader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C,MAAM,CAAC,OAAO,OAAO,YAAY;IAC/B,OAAc,MAAM,EAAE,MAAM,CAAC;WAEf,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;CAOvE"}

package/dist/ConfigReader.js

@@ -0,0 +1,51 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const yaml_1 = __importDefault(require("yaml"));
+const fs = __importStar(require("fs"));
+class ConfigReader {
+    static generate(configPath, pluginPath) {
+        ConfigReader.config = {
+            ...yaml_1.default.parse(fs.readFileSync(configPath, 'utf8')),
+            ...yaml_1.default.parse(fs.readFileSync(pluginPath, 'utf8'))
+        };
+        return ConfigReader.config;
+    }
+}
+exports.default = ConfigReader;
+//# sourceMappingURL=ConfigReader.js.map

package/dist/ConfigReader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConfigReader.js","sourceRoot":"","sources":["../src/ConfigReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AACxB,uCAAyB;AAGzB,MAAqB,YAAY;IAGxB,MAAM,CAAC,QAAQ,CAAC,UAAkB,EAAE,UAAkB;QAC3D,YAAY,CAAC,MAAM,GAAG;YACpB,GAAG,cAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAClD,GAAG,cAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;SACnD,CAAC;QACF,OAAO,YAAY,CAAC,MAAM,CAAC;IAC7B,CAAC;CACF;AAVD,+BAUC"}

package/dist/PipelineStage.d.ts

@@ -0,0 +1,19 @@
+import { Kafka, Producer, Consumer } from "kafkajs";
+import PipelinePluginI from "./plugin/PipelinePluginI";
+import { Config } from "@cdp-forge/types";
+export default class PipelineSTage {
+    plugin: PipelinePluginI;
+    consumer: Consumer;
+    producer: Producer;
+    kafka: Kafka;
+    input: string | null;
+    output: string | null;
+    currentOperation: Promise<void>;
+    constructor(plugin: PipelinePluginI, config: Config);
+    start(inputTopic: string, outputTopic?: string | null): Promise<void>;
+    private _start;
+    stop(): Promise<void>;
+    private _stop;
+    restart(inputTopic: string, outputTopic: string | null): Promise<void>;
+}
+//# sourceMappingURL=PipelineStage.d.ts.map

package/dist/PipelineStage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PipelineStage.d.ts","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAsB,MAAM,SAAS,CAAC;AACxE,OAAO,eAAe,MAAM,0BAA0B,CAAC;AACvD,OAAO,EAAO,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAI/C,MAAM,CAAC,OAAO,OAAO,aAAa;IAC9B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEpB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;IAa7C,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,IAAI,CAAC;YASnE,MAAM;IA4Bd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAMb,KAAK;IAKb,OAAO,CAAC,UAAU,EAAE,MAAM,EAAC,WAAW,EAAE,MAAM,GAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CAI5E"}

package/dist/PipelineStage.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkajs_1 = require("kafkajs");
+const podName = process.env.CLIENT_ID || 'default-client-id';
+class PipelineSTage {
+    constructor(plugin, config) {
+        this.plugin = plugin;
+        this.kafka = new kafkajs_1.Kafka({
+            clientId: config.plugin.name + `plugin-${podName}`,
+            brokers: config.kafkaConfig.brokers,
+        });
+        this.consumer = this.kafka.consumer({ groupId: config.plugin.name + `plugin` });
+        this.producer = this.kafka.producer();
+        this.input = null;
+        this.output = null;
+        this.currentOperation = Promise.resolve();
+    }
+    async start(inputTopic, outputTopic = null) {
+        this.input = inputTopic;
+        this.output = outputTopic;
+        this.currentOperation = this.currentOperation.then(() => this.plugin.init());
+        await this.currentOperation;
+        this.currentOperation = this._start();
+        return this.currentOperation;
+    }
+    async _start() {
+        if (this.output)
+            await this.producer.connect();
+        await this.consumer.connect();
+        await this.consumer.subscribe({ topic: this.input, fromBeginning: false });
+        await this.consumer.run({
+            autoCommit: false,
+            eachMessage: async ({ topic, partition, message }) => {
+                const log = message.value ? JSON.parse(message.value.toString()) : null;
+                if (!log)
+                    return;
+                const elaboratedLog = await this.plugin.elaborate(log);
+                if (!elaboratedLog) {
+                    console.warn('Messaggio non elaborato');
+                    await this.consumer.commitOffsets([{ topic, partition, offset: (BigInt(message.offset) + 1n).toString() }]);
+                    return;
+                }
+                if (this.output) {
+                    await this.producer.send({
+                        topic: this.output,
+                        messages: [{ value: JSON.stringify(elaboratedLog) }],
+                    });
+                    await this.consumer.commitOffsets([{ topic, partition, offset: (BigInt(message.offset) + 1n).toString() }]);
+                }
+            },
+        });
+    }
+    async stop() {
+        await this.currentOperation;
+        this.currentOperation = this._stop();
+        return this.currentOperation;
+    }
+    async _stop() {
+        await this.consumer.disconnect();
+        await this.producer.disconnect();
+    }
+    async restart(inputTopic, outputTopic) {
+        await this.stop();
+        await this.start(inputTopic, outputTopic);
+    }
+}
+exports.default = PipelineSTage;
+//# sourceMappingURL=PipelineStage.js.map

package/dist/PipelineStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PipelineStage.js","sourceRoot":"","sources":["../src/PipelineStage.ts"],"names":[],"mappings":";;AAAA,qCAAwE;AAIxE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,mBAAmB,CAAC;AAE7D,MAAqB,aAAa;IAS9B,YAAY,MAAuB,EAAE,MAAc;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,UAAU,OAAO,EAAE;YAClD,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,cAA6B,IAAI;QAC7D,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,MAAM;QAChB,IAAG,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAE9C,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACtB,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAuB,EAAE,EAAE;gBACxE,MAAM,GAAG,GAAQ,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC7E,IAAG,CAAC,GAAG;oBAAE,OAAO;gBAChB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACvD,IAAG,CAAC,aAAa,EAAE,CAAC;oBAClB,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;oBAC5G,OAAO;gBACT,CAAC;gBAED,IAAG,IAAI,CAAC,MAAM,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACvB,KAAK,EAAE,IAAI,CAAC,MAAM;wBAClB,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;qBACnD,CAAC,CAAC;oBACL,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC9G,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,gBAAgB,CAAC;QAC5B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,UAAkB,EAAC,WAAwB;QACrD,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;CACJ;AA1ED,gCA0EC"}

package/dist/config/config.yml

@@ -0,0 +1,7 @@
+# This file is generated by the helm package and should be mounted on the plugin container
+kafkaConfig:
+  brokers:
+  -  'localhost:36715'
+  config_topic: 'config'
+
+pluginManagerUrl: 'https://plugin_template_url'

package/dist/config/plugin.yml

@@ -0,0 +1,4 @@
+plugin:
+  name: 'myPlugin'
+  priority: 1 # 1 to 100 (no need if parallel)
+  type: 'blocking' # or 'parallel'

package/dist/config.d.ts

@@ -0,0 +1,8 @@
+export default class Config {
+    private static instance;
+    private readonly configData;
+    private constructor();
+    static getInstance(): Config;
+    get config(): any;
+}
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,OAAO,OAAO,MAAM;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAM;IAEjC,OAAO;WAYO,WAAW,IAAI,MAAM;IAOnC,IAAW,MAAM,IAAI,GAAG,CAEvB;CACF"}