@5minds/node-red-contrib-processcube-elasticsearch 0.4.0 → 1.0.0-develop-31b02f-m6dsjh20

package/.processcube/nodered/package.json

@@ -5,9 +5,11 @@
     "private": true,
     "dependencies": {
         "@flowfuse/node-red-dashboard": "1.22.0",
-        "passport-oauth2": "*",
+        "do-red": "~1.0.7",
         "jsonwebtoken": "*",
         "jwks-rsa": "*",
-        "node-red-contrib-oauth2": "*"
+        "node-red-contrib-oauth2": "*",
+        "node-red-contrib-opentelemetry": "~1.6.2",
+        "passport-oauth2": "*"
     }
-}
+}

package/Dockerfile

@@ -9,6 +9,9 @@ RUN cd /package_src/ && npm install
 
 RUN npm install /package_src/
 
+# test pckages
+RUN npm install do-red
+
 # defaults
 USER node-red
 

package/docker-compose.yml

@@ -18,13 +18,12 @@ services:
       - ./.processcube/nodered:/data
   
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.1
+    image: elasticsearch:8.16.3
     container_name: elasticsearch
     environment:
       - discovery.type=single-node
       - xpack.security.enabled=false
       - xpack.security.http.ssl.enabled=false
-     # - ELASTIC_PASSWORD=myelasticpassword
       - ES_JAVA_OPTS=-Xms512m -Xmx512m
     ports:
       - "9200:9200"
@@ -33,6 +32,32 @@ services:
       - es_data:/usr/share/elasticsearch/data
     restart: always
 
+  apm-server:
+    image: elastic/apm-server:8.16.3
+    container_name: apm-server
+    environment:
+      - output.elasticsearch.hosts=["http://elasticsearch:9200"]
+      - apm-server.host=0.0.0.0:8200
+      - apm-server.enable=true
+      - apm-server.instrumentation.enabled=true
+      - apm-server.instrumentation.otlp.enabled=true # Aktiviert OpenTelemetry
+      - apm-server.instrumentation.otlp.http.endpoint=/v1/traces    
+    ports:
+      - "8200:8200" # OpenTelemetry-Endpunkt
+    depends_on:
+      - elasticsearch
+
+  kibana:
+    image: kibana:8.16.3
+    container_name: kibana
+    environment:
+      - xpack.security.enabled=false
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    ports:
+      - "5601:5601"
+    depends_on:
+      - elasticsearch
+
 volumes:
   es_data:
     driver: local

package/elastic-search-logger.js

@@ -2,55 +2,6 @@
 
 const winston = require('winston');
 const winstonElasticSearch = require('winston-elasticsearch');
-const { ElasticsearchTransformer } = require('winston-elasticsearch');
-
-const logLevels = {
-    levels: {
-        Error: 0,
-        Warning: 1,
-        Information: 2,
-        Debug: 3
-    }
-};
-
-function transformElasticFields(logData) {
-    const transformed = ElasticsearchTransformer(logData);
-
-    transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString();
-    transformed.message = logData.message;
-    transformed.messageTemplate = logData.messageTemplate;
-    transformed.severity = logData.level;
-    transformed.level = logData.level;
-    transformed.fields = logData.meta;
-
-    if (logData.meta['transaction.id']) {
-        transformed.transaction = { id: logData.meta['transaction.id'] };
-    }
-
-    if (logData.meta['trace.id']) {
-        transformed.trace = {
-            id: logData.meta['trace.id']
-        };
-    }
-
-    if (logData.meta['span.id']) {
-        transformed.span = {
-            id: logData.meta['span.id']
-        };
-    }
-
-    return transformed;
-}
-
-function raiseErrorAndSetNodeStatus(node, message) {
-    node.error(message, {});
-    node.status({
-        fill: 'red',
-        shape: 'ring',
-        text: message,
-    });
-}
-
 
 // TODO: MM winston-elasticsearch/index.js#112 wird die log als async function deklariert, 
 // aber nicht als async aufgerufen
@@ -58,9 +9,26 @@ process.on('unhandledRejection', (reason, promise) => {
     console.error(`Unhandled Rejection at ${promise} reason: ${reason}`, {});
 });    
 
-
 module.exports = function (RED) {
-
+    
+    const logLevels = {
+        levels: {
+            Error: 0,
+            Warning: 1,
+            Information: 2,
+            Debug: 3
+        }
+    };
+    
+    function raiseErrorAndSetNodeStatus(node, message) {
+        node.error(message, {});
+        node.status({
+            fill: 'red',
+            shape: 'ring',
+            text: message,
+        });
+    }
+        
     function LogElasticLoggerNode(config) {
         RED.nodes.createNode(this, config);
         const node = this;
@@ -91,66 +59,39 @@ module.exports = function (RED) {
         }
 
         if (url) {
-            const transports = [];
-            const elasticSearchTransport = new winstonElasticSearch.ElasticsearchTransport({
-                clientOpts: {
-                    node: url,
-                    auth: {
-                        username: user,
-                        password: password,
-                    },
-                    ssl: {
-                        // accept any
-                        rejectUnauthorized: false,
-                    },
-                },
-                transformer: (logData) => transformElasticFields(logData),
-                index: index,
-            });
+            node._url = `${url}/${index}/_doc`;
 
-            transports.push(elasticSearchTransport);
+            if (user && password) {
+                node._credentials = Buffer.from(`${user}:${password}`).toString('base64');
+            }
 
-            elasticSearchTransport.on('error', (error) => {
-                raiseErrorAndSetNodeStatus(node, `Error in elasticSearchTransport caught: ${error.message}`);
-            });
+        }
 
-            node.logger = new winston.createLogger({
-                exitOnError: false,
-                level: 'Debug',
-                levels: logLevels.levels,
-                transports: transports,
-            });
+        node.addToLog = async (record) => {
+            const headers = {
+                'Content-Type': 'application/json',
+            };
+            
+            if (node._credentials) {
+                headers.Authorization = `Basic ${node._credentials}`;
+            }
 
-            node.logger.on('error', (error) => {
-                raiseErrorAndSetNodeStatus(node, `Error in logger caught: ${error.message}`);
+            const response = await fetch(node._url, {
+                method: 'POST',
+                headers,
+                body: JSON.stringify(record),
             });
 
-            this.debug('elastic-search logger created');
-        }
-
+            const result = await response.json();
+            
+            return result;
+        };
 
         this.on('close', function () {
-            // close logger
-            if (node.logger) {
-                node.logger.close();
-            }
-
             node.debug('elastic-search logger closed');
         });
     }
 
-    LogElasticLoggerNode.prototype.addToLog = function addTolog(loglevel, msg) {
-        try {
-            if (this.logger == null) {
-                raiseErrorAndSetNodeStatus(this, `Elastic search logger is not initialized: ${JSON.stringify(msg)}`);
-            } else {
-                this.logger.log(loglevel, msg.payload.message, msg.payload.meta);
-            }
-        } catch (error) {
-            this.error(error, msg);
-        }
-    };
-
     RED.nodes.registerType('elastic-search-logger', LogElasticLoggerNode, {
         credentials: {
             username: { type: 'text' },

package/elastic-search.html

@@ -17,7 +17,7 @@
       }
     },
     inputs: 1,
-    outputs: 0,
+    outputs: 1,
     icon: "file.svg",
     align: "right",
     label: function () {
@@ -100,15 +100,20 @@
     </div>
 </script>
 
-<script type="text/x-red" data-help-name="elastic-search">
-    <p>A logging node for Elastic Search using the winston and winston-elastic logging libraries</p>
-    <h3>Details</h3>
-    <p>Output <code>msg.payload</code> (or <code>msg.&lt;custom path&gt;</code>) or complete <code>msg</code> is used as input of the logged message.</p>
-    <p>The Log Level (Error, Warning, Information, Debug) used can be configured in the node or can be set
-    by <code>msg.loglevel</code> (or <code>msg.&lt;custom path&gt;</code>). 
-    If the content is not one of ('Error', 'Warning', 'Information', 'Debug'), 'Debug' will be used as fallback</p>
-    <p>If <code>msg.meta</code> is set, the meta info Object will be added to the file and ElasticSearch output as a JSON string.</p>
+<script type="text/markdown" data-help-name="elastic-search">
+Sending Data to the Elastic Search.
 
-    <h3>References</h3>
-    <p>This node is based on parts of node-red-contrib-advance-logger</p>
-</script>
+## Inputs
+
+: payload (object) : Will mapped to the record for elastic and meta-data are mapped to the key `fields`
+
+## Outputs
+
+: payload (string) : The result of the elastic call.
+
+
+### References
+
+-  [The ProcessCube&copy; Developer Network](https://processcube.io) - All documentation for the ProcessCube&copy; platform
+-  [ProcessCube&copy; LowCode Integration](https://processcube.io/docs/node-red) - LowCode integration in ProcessCube&copy;
+  </script>

package/elastic-search.js

@@ -6,11 +6,45 @@ module.exports = function (RED) {
     const Information = 'Information';
     const Debug = 'Debug';
 
+    function mapMessage(msg) {
+        const transformed = {};
+
+        transformed['@timestamp'] = msg.payload.timestamp ? msg.timestamp : new Date().toISOString();
+        transformed.message = msg.payload.message;
+        transformed.messageTemplate = msg.payload.messageTemplate;
+        transformed.severity = msg.payload.level;
+        transformed.level = msg.payload.level;
+        let meta = msg.payload.meta;
+
+        if (meta['transaction.id']) {
+            transformed.transaction = { id: meta['transaction.id'] };
+
+            delete meta['transaction.id'];
+        }
+
+        if (msg.payload.meta['trace.id']) {
+            transformed.trace = { id: meta['trace.id'] };
+
+            delete meta['trace.id'];
+        }
+
+        if (msg.payload.meta['span.id']) {
+            transformed.span = { id: meta['span.id'] };
+
+            delete meta['span.id'];
+        }
+
+        transformed.fields = meta;
+
+        return transformed;
+    }
+
+
     function LogElasticNode(config) {
         RED.nodes.createNode(this, config);
         const node = this;
 
-        this.on('input', function (msg) {
+        this.on('input', async function (msg) {
             node.logger = RED.nodes.getNode(config.logger);
 
             let loglevel = config.loglevel || '';
@@ -35,7 +69,14 @@ module.exports = function (RED) {
                 }
 
                 try {
-                    node.logger.addToLog(level, msg);
+
+                    const mappedMessage = mapMessage(msg);
+
+                    const result = await node.logger.addToLog(mappedMessage);
+
+                    msg.payload = result;
+
+                    node.send(msg);
                 } catch (err) {
                     node.error(err);
                 }

package/examples/elastic-search-example.json

@@ -0,0 +1,222 @@
+[
+    {
+        "id": "53b75a173ac5abcc",
+        "type": "tab",
+        "label": "Elasticsearch loggen",
+        "disabled": false,
+        "info": "",
+        "env": []
+    },
+    {
+        "id": "38edcaecf5968ff5",
+        "type": "group",
+        "z": "53b75a173ac5abcc",
+        "style": {
+            "stroke": "#999999",
+            "stroke-opacity": "1",
+            "fill": "none",
+            "fill-opacity": "1",
+            "label": true,
+            "label-position": "nw",
+            "color": "#a4a4a4"
+        },
+        "nodes": [
+            "0b2f670fec931f5f",
+            "443b329ab591c4dd",
+            "f4a0d7de5d04c121",
+            "946add545fa1db9f",
+            "0c15060915aac994"
+        ],
+        "x": 34,
+        "y": 19,
+        "w": 372,
+        "h": 202
+    },
+    {
+        "id": "4aae454884faa826",
+        "type": "group",
+        "z": "53b75a173ac5abcc",
+        "style": {
+            "stroke": "#999999",
+            "stroke-opacity": "1",
+            "fill": "none",
+            "fill-opacity": "1",
+            "label": true,
+            "label-position": "nw",
+            "color": "#a4a4a4"
+        },
+        "nodes": [
+            "f90f2ba54f023834",
+            "31dcfc340dae3780",
+            "59df2485b6c11412"
+        ],
+        "x": 34,
+        "y": 259,
+        "w": 372,
+        "h": 142
+    },
+    {
+        "id": "0b2f670fec931f5f",
+        "type": "elastic-search",
+        "z": "53b75a173ac5abcc",
+        "g": "38edcaecf5968ff5",
+        "name": "",
+        "logger": "e8672d361683672d",
+        "complete": "payload",
+        "loglevel": "Debug",
+        "x": 300,
+        "y": 120,
+        "wires": [
+            []
+        ]
+    },
+    {
+        "id": "443b329ab591c4dd",
+        "type": "inject",
+        "z": "53b75a173ac5abcc",
+        "g": "38edcaecf5968ff5",
+        "name": "",
+        "props": [
+            {
+                "p": "payload"
+            }
+        ],
+        "repeat": "",
+        "crontab": "",
+        "once": false,
+        "onceDelay": 0.1,
+        "topic": "",
+        "payload": "{\"message\":\"hello world\",\"messageTemplate\":\"a messageTemplate\",\"severity\":\"a severity\",\"level\":\"a level\",\"meta\":{\"firstField\":\"a FirstField\",\"secondField\":\"a SecondField\",\"transaction.id\":\"a transaction.id\",\"trace.id\":\"a trace.id\",\"span.id\":\"a span.id\"}}",
+        "payloadType": "json",
+        "x": 130,
+        "y": 120,
+        "wires": [
+            [
+                "0b2f670fec931f5f"
+            ]
+        ]
+    },
+    {
+        "id": "f4a0d7de5d04c121",
+        "type": "comment",
+        "z": "53b75a173ac5abcc",
+        "g": "38edcaecf5968ff5",
+        "name": "Handle log",
+        "info": "",
+        "x": 120,
+        "y": 60,
+        "wires": []
+    },
+    {
+        "id": "946add545fa1db9f",
+        "type": "catch",
+        "z": "53b75a173ac5abcc",
+        "g": "38edcaecf5968ff5",
+        "name": "",
+        "scope": null,
+        "uncaught": false,
+        "x": 120,
+        "y": 180,
+        "wires": [
+            [
+                "0c15060915aac994"
+            ]
+        ]
+    },
+    {
+        "id": "0c15060915aac994",
+        "type": "debug",
+        "z": "53b75a173ac5abcc",
+        "g": "38edcaecf5968ff5",
+        "name": "error",
+        "active": true,
+        "tosidebar": true,
+        "console": false,
+        "tostatus": false,
+        "complete": "true",
+        "targetType": "full",
+        "statusVal": "",
+        "statusType": "auto",
+        "x": 270,
+        "y": 180,
+        "wires": []
+    },
+    {
+        "id": "f90f2ba54f023834",
+        "type": "elastic-search",
+        "z": "53b75a173ac5abcc",
+        "g": "4aae454884faa826",
+        "name": "",
+        "logger": "3667972f5cb77061",
+        "complete": "payload",
+        "loglevel": "Debug",
+        "x": 300,
+        "y": 360,
+        "wires": [
+            []
+        ]
+    },
+    {
+        "id": "31dcfc340dae3780",
+        "type": "inject",
+        "z": "53b75a173ac5abcc",
+        "g": "4aae454884faa826",
+        "name": "",
+        "props": [
+            {
+                "p": "payload"
+            }
+        ],
+        "repeat": "",
+        "crontab": "",
+        "once": false,
+        "onceDelay": 0.1,
+        "topic": "",
+        "payload": "{\"message\":\"hello world\"}",
+        "payloadType": "json",
+        "x": 130,
+        "y": 360,
+        "wires": [
+            [
+                "f90f2ba54f023834"
+            ]
+        ]
+    },
+    {
+        "id": "59df2485b6c11412",
+        "type": "comment",
+        "z": "53b75a173ac5abcc",
+        "g": "4aae454884faa826",
+        "name": "Handle failed log",
+        "info": "",
+        "x": 140,
+        "y": 300,
+        "wires": []
+    },
+    {
+        "id": "e8672d361683672d",
+        "type": "elastic-search-logger",
+        "name": "Docker compose",
+        "url": "http://elasticsearch:9200",
+        "urlType": "str",
+        "usernameType": "str",
+        "passwordType": "str",
+        "indexType": "str",
+        "filename": "log-elastic.log",
+        "maxsize": 1,
+        "maxfiles": 2
+    },
+    {
+        "id": "3667972f5cb77061",
+        "type": "elastic-search-logger",
+        "name": "Failed docker compose",
+        "url": "",
+        "urlType": "str",
+        "usernameType": "str",
+        "passwordType": "str",
+        "indexType": "str",
+        "filename": "log-elastic.log",
+        "maxsize": 1,
+        "maxfiles": 2
+    }
+]

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@5minds/node-red-contrib-processcube-elasticsearch",
-    "version": "0.4.0",
+    "version": "1.0.0-develop-31b02f-m6dsjh20",
     "license": "MIT",
     "description": "Node-RED nodes for Elasticsearch",
     "scripts": {
@@ -36,10 +36,6 @@
         },
         "examples": "examples"
     },
-    "dependencies": {
-        "winston": "^3.17.0",
-        "winston-elasticsearch": "^0.19.0"
-    },
     "keywords": [
         "node-red",
         "processcube",