@5minds/node-red-contrib-processcube-tools 1.2.0-feature-7ac247-mg92hchl → 1.2.0-feature-608421-mg9cjskq

package/file-storage/file-storage.html

@@ -0,0 +1,203 @@
+<script type="text/javascript">
+    RED.nodes.registerType('file-storage', {
+        category: 'ProcessCube Tools',
+        color: '#02AFD6',
+        defaults: {
+            name: { value: '' },
+            provider: { value: 'fs' },
+            baseDir: { value: '' },
+            usernameType: { value: 'str' },
+            username: { value: '', required: true, validate: RED.validators.typedInput('usernameType') },
+            passwordType: { value: 'str' },
+            password: { value: '', required: true, validate: RED.validators.typedInput('passwordType') },
+            hostType: { value: 'str' },
+            host: { value: '', required: true, validate: RED.validators.typedInput('hostType') },
+            portType: { value: 'str' },
+            port: { value: '', required: true, validate: RED.validators.typedInput('portType') },
+            databaseType: { value: 'str' },
+            database: { value: '', required: true, validate: RED.validators.typedInput('databaseType') },
+            
+            pgSchema: { value: 'public' },
+            pgTable: { value: 'files' },
+            outputAs: { value: 'stream' },
+            defaultAction: { value: 'store' },
+        },
+        inputs: 1,
+        outputs: 1,
+        icon: 'file.png',
+        label: function () {
+            return this.name || 'file-storage';
+        },
+        oneditprepare: function () {
+            // postgres fields user, password, host, port, database
+            $('#node-input-username').typedInput({
+                default: 'str',
+                types: ['str', 'env'],
+                typeField: '#node-input-usernameType',
+            });
+            $('#node-input-password').typedInput({
+                default: 'str',
+                types: ['str', 'env'],
+                typeField: '#node-input-passwordType',
+            });
+            $('#node-input-host').typedInput({
+                default: 'str',
+                types: ['str', 'env'],
+                typeField: '#node-input-hostType',
+            });
+            $('#node-input-port').typedInput({
+                default: 'str',
+                types: ['str', 'env'],
+                typeField: '#node-input-portType',
+            });
+            $('#node-input-database').typedInput({
+                default: 'str',
+                types: ['str', 'env'],
+                typeField: '#node-input-databaseType',
+            });
+        },
+    });
+</script>
+
+<script type="text/html" data-template-name="file-storage">
+    <div class="form-row">
+        <label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
+        <input type="text" id="node-input-name" placeholder="file-storage" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-provider"><i class="fa fa-database"></i> Provider</label>
+        <select id="node-input-provider">
+            <option value="fs">Filesystem</option>
+            <option value="pg">PostgreSQL</option>
+        </select>
+    </div>
+    <div class="form-row">
+        <label for="node-input-outputAs"><i class="fa fa-share-square-o"></i> Output</label>
+        <select id="node-input-outputAs">
+            <option value="stream">Stream</option>
+            <option value="buffer">Buffer</option>
+            <option value="path">Path (nur FS)</option>
+        </select>
+    </div>
+    <hr />
+    <div class="form-tips">Filesystem</div>
+    <div class="form-row">
+        <label for="node-input-baseDir"><i class="fa fa-folder-open"></i> Base Dir</label>
+        <input type="text" id="node-input-baseDir" placeholder="/data/files" />
+    </div>
+    <hr />
+    <div class="form-tips">PostgreSQL</div>
+    <div class="form-row">
+        <label for="node-input-username"><i class="fa fa-user"></i> Username</label>
+        <input type="text" id="node-input-username" placeholder="postgres" />
+        <input type="hidden" id="node-input-usernameType" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-password"><i class="fa fa-key"></i> Password</label>
+        <input type="text" id="node-input-password" placeholder="postgres" />
+         <input type="hidden" id="node-input-passwordType" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-host"><i class="fa fa-server"></i> Host</label>
+        <input type="text" id="node-input-host" placeholder="localhost" />
+         <input type="hidden" id="node-input-hostType" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-port"><i class="fa fa-plug"></i> Port</label>
+        <input type="text" id="node-input-port" placeholder="5432" />
+         <input type="hidden" id="node-input-portType" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-database"><i class="fa fa-database"></i> Database</label>
+        <input type="text" id="node-input-database" placeholder="postgres" />
+         <input type="hidden" id="node-input-databaseType" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-pgSchema"><i class="fa fa-sitemap"></i> Schema</label>
+        <input type="text" id="node-input-pgSchema" placeholder="public" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-pgTable"><i class="fa fa-table"></i> Table</label>
+        <input type="text" id="node-input-pgTable" placeholder="files" />
+    </div>
+    <hr />
+    <div class="form-row">
+        <label for="node-input-defaultAction"><i class="fa fa-cog"></i> Default Action</label>
+        <select id="node-input-defaultAction">
+            <option value="store">store</option>
+            <option value="get">get</option>
+            <option value="delete">delete</option>
+        </select>
+    </div>
+</script>
+
+<script type="text/html" data-help-name="file-storage">
+    <h2>File Storage Node</h2>
+    <p>
+        A Node-RED node for storing, retrieving, and deleting files (including metadata) using either the local filesystem or PostgreSQL (Large Objects + metadata table) as a backend.
+    </p>
+    <h3>Features</h3>
+    <ul>
+        <li><b>Providers:</b>
+            <ul>
+                <li>Filesystem (stores file and metadata as JSON)</li>
+                <li>PostgreSQL (stores file as Large Object and metadata in a table)</li>
+            </ul>
+        </li>
+        <li><b>Actions:</b> Store, Retrieve, Delete files</li>
+        <li><b>Flexible output:</b> Stream, Buffer, or Path (filesystem only)</li>
+    </ul>
+    <h3>Node Properties</h3>
+    <ul>
+        <li><b>Name:</b> Optional node label.</li>
+        <li><b>Provider:</b> Select between Filesystem and PostgreSQL.</li>
+        <li><b>Output:</b> Choose the output type for retrieval: Stream, Buffer, or Path (Path only for filesystem).</li>
+        <li><b>Base Dir:</b> (Filesystem) Directory where files are stored.</li>
+        <li><b>Schema:</b> (PostgreSQL) Database schema (default: public).</li>
+        <li><b>Table:</b> (PostgreSQL) Table for metadata (default: files).</li>
+        <li><b>Default Action:</b> Default action if not specified in the message (store, get, or delete).</li>
+    </ul>
+    <h3>Input</h3>
+    <pre>
+msg.action = "store" | "get" | "delete" // Optional, overrides defaultAction
+msg.payload = Buffer | ReadableStream | String // For "store"
+msg.file = {
+  id?: string,         // For "get" or "delete"
+  filename?: string,   // For "store"
+  contentType?: string,// For "store"
+  metadata?: object    // For "store"
+}
+    </pre>
+    <h3>Output</h3>
+    <ul>
+        <li><b>store:</b> <code>msg.payload</code> contains metadata including the generated <code>id</code>. <code>msg.file</code> contains the merged file info and result.</li>
+        <li><b>get:</b> <code>msg.payload</code> contains the file as a Stream, Buffer, or Path (depending on output setting). <code>msg.file</code> contains the file metadata.</li>
+        <li><b>delete:</b> <code>msg.payload</code> contains the result of the delete operation.</li>
+    </ul>
+    <h3>Example Usage</h3>
+    <pre>
+// Store a file:
+msg.action = "store";
+msg.payload = Buffer.from("Hello World");
+msg.file = {
+  filename: "hello.txt",
+  contentType: "text/plain",
+  metadata: { author: "Alice" }
+};
+
+// Retrieve a file:
+msg.action = "get";
+msg.file = { id: "your-file-id" };
+
+// Delete a file:
+msg.action = "delete";
+msg.file = { id: "your-file-id" };
+    </pre>
+    <h3>Notes</h3>
+    <ul>
+        <li>For PostgreSQL, ensure the connection string, schema, and table exist and the user has the necessary permissions.</li>
+        <li>For filesystem storage, ensure the base directory is writable by Node-RED.</li>
+        <li>The node is designed to handle large files efficiently using streams.</li>
+    </ul>
+    <p><b>Enjoy using the File Storage Node in your Node-RED flows!</b></p>
+</script>

package/file-storage/file-storage.js

@@ -0,0 +1,148 @@
+/**
+ * # File Storage Node
+ *
+ * A Node-RED node for storing, retrieving, and deleting files (including metadata) using either the local filesystem or PostgreSQL (Large Objects + metadata table) as a backend.
+ *
+ * ## Features
+ * - **Providers:**
+ *   - Filesystem (stores file and metadata as JSON)
+ *   - PostgreSQL (stores file as Large Object and metadata in a table)
+ * - **Actions:**
+ *   - Store a file
+ *   - Retrieve a file
+ *   - Delete a file
+ * - **Flexible output:**
+ *   - Stream, Buffer, or Path (filesystem only)
+ *
+ * ## Node Properties
+ * - **Name:** Optional node label.
+ * - **Provider:** Select between `Filesystem` and `PostgreSQL`.
+ * - **Output:** Choose the output type for retrieval: `Stream`, `Buffer`, or `Path` (Path only for filesystem).
+ * - **Base Dir:** (Filesystem) Directory where files are stored.
+ * - **Connection:** (PostgreSQL) Connection string for the database.
+ * - **Schema:** (PostgreSQL) Database schema (default: `public`).
+ * - **Table:** (PostgreSQL) Table for metadata (default: `files`).
+ * - **Default Action:** Default action if not specified in the message (`store`, `get`, or `delete`).
+ *
+ * ## Input
+ * The node expects the following properties in the incoming message:
+ * ```
+ * msg.action = "store" | "get" | "delete" // Optional, overrides defaultAction
+ * msg.payload = Buffer | ReadableStream | String // For "store"
+ * msg.file = {
+ *   id?: string,         // For "get" or "delete"
+ *   filename?: string,   // For "store"
+ *   contentType?: string,// For "store"
+ *   metadata?: object    // For "store"
+ * }
+ * ```
+ *
+ * ## Output
+ * - For **store**:
+ *   - `msg.payload` contains metadata including the generated `id`.
+ *   - `msg.file` contains the merged file info and result.
+ * - For **get**:
+ *   - `msg.payload` contains the file as a Stream, Buffer, or Path (depending on output setting).
+ *   - `msg.file` contains the file metadata.
+ * - For **delete**:
+ *   - `msg.payload` contains the result of the delete operation.
+ *
+ * ## Example Usage
+ * // Store a file:
+ * msg.action = "store";
+ * msg.payload = Buffer.from("Hello World");
+ * msg.file = {
+ *   filename: "hello.txt",
+ *   contentType: "text/plain",
+ *   metadata: { author: "Alice" }
+ * };
+ *
+ * // Retrieve a file:
+ * msg.action = "get";
+ * msg.file = { id: "your-file-id" };
+ *
+ * // Delete a file:
+ * msg.action = "delete";
+ * msg.file = { id: "your-file-id" };
+ *
+ * ## Notes
+ * - For PostgreSQL, ensure the connection string, schema, and table exist and the user has the necessary permissions.
+ * - For filesystem storage, ensure the base directory is writable by Node-RED.
+ * - The node is designed to handle large files efficiently using streams.
+ *
+ * Enjoy using the File Storage Node in your Node-RED flows!
+ */
+module.exports = function (RED) {
+    const StorageCore = require('../storage/storage-core');
+
+    function FileStorageNode(config) {
+        RED.nodes.createNode(this, config);
+        const node = this;
+
+        // Node-Konfiguration
+        node.provider = config.provider || 'fs';
+        node.baseDir = config.baseDir;
+        node.pg = {
+            username:  RED.util.evaluateNodeProperty(config.username, config.usernameType, node) || 'postgres',
+            password: RED.util.evaluateNodeProperty(config.password, config.passwordType, node) || 'postgres',
+            host: RED.util.evaluateNodeProperty(config.host, config.hostType, node) || 'localhost',
+            port: RED.util.evaluateNodeProperty(config.port, config.portType, node) || 5432,
+            database: RED.util.evaluateNodeProperty(config.database, config.databaseType, node) || 'postgres',
+            schema: config.pgSchema || 'public',
+            table: config.pgTable || 'files',
+        };
+        node.outputAs = config.outputAs || 'stream'; // 'stream' | 'buffer' | 'path' (path nur fs)
+
+        // Storage-Kern
+        const storage = new StorageCore({
+            provider: node.provider,
+            fs: { baseDir: node.baseDir },
+            pg: node.pg,
+        });
+
+        storage.init().catch((err) => node.error(err));
+
+        node.on('input', async function (msg, send, done) {
+            try {
+                const action = msg.action || config.defaultAction || 'store';
+                if (action === 'store') {
+                    const file = msg.file || {};
+                    const result = await storage.store(msg.payload, file);
+                    msg.payload = result;
+                    msg.file = { ...file, ...result };
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                if (action === 'get') {
+                    const id = msg.file && msg.file.id;
+                    if (!id) throw new Error('file.id is required for get');
+                    const { meta, payload } = await storage.get(id, { as: node.outputAs });
+                    msg.file = { ...meta, id: meta.id };
+                    msg.payload = payload;
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                if (action === 'delete') {
+                    const id = msg.file && msg.file.id;
+                    if (!id) throw new Error('file.id is required for delete');
+                    const result = await storage.delete(id);
+                    msg.payload = result;
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                throw new Error(`Unknown action: ${action}`);
+            } catch (err) {
+                node.error(err, msg);
+                if (done) done(err);
+            }
+        });
+    }
+
+    RED.nodes.registerType('file-storage', FileStorageNode);
+};

package/package.json

@@ -1,16 +1,11 @@
 {
     "name": "@5minds/node-red-contrib-processcube-tools",
-    "version": "1.2.0-feature-7ac247-mg92hchl",
+    "version": "1.2.0-feature-608421-mg9cjskq",
     "license": "MIT",
     "description": "Node-RED tools nodes for ProcessCube",
     "scripts": {
-        "build": "npm-run-all build:*",
-        "build:ts": "tsc",
-        "build:html": "cpx \"./src/**/*.html\" ./dist",
-        "lint": "prettier --write --config ./.prettierrc.json \"**/*.{html,ts}\"",
-        "test": "mocha --require ts-node/register 'src/test/**/*.test.ts'",
-        "test:unit": "mocha --require ts-node/register 'src/test/unit/**/*.test.ts'",
-        "test:integration": "mocha --require ts-node/register 'src/test/integration/**/*.test.ts'"
+        "lint": "prettier --write --config ./.prettierrc.json \"**/*.{html,js}\"",
+        "test": "mocha test/unit/ test/integration"
     },
     "authors": [
         {
@@ -24,6 +19,10 @@
         {
             "name": "Diana Stefan",
             "email": "Diana.Stefan@5Minds.de"
+        },
+        {
+            "name": "Thorsten Kallweit",
+            "email": "Thorsten.Kallweit@5Minds.de"
         }
     ],
     "repository": {
@@ -39,43 +38,35 @@
         "npm": ">=8.0.0"
     },
     "node-red": {
-        "version": "^4.1.0",
+        "version": ">=3.1.9",
         "nodes": {
-            "EmailReceiver": "dist/email-receiver/email-receiver.js",
-            "EmailSender": "dist/email-sender/email-sender.js",
-            "HtmlToText": "dist/html-to-text/html-to-text.js"
+            "EmailReceiver": "email-receiver/email-receiver.js",
+            "EmailSender": "email-sender/email-sender.js",
+            "HtmlToText": "processcube-html-to-text/processcube-html-to-text.js",
+            "FileStorage": "file-storage/file-storage.js"
         },
         "examples": "examples"
     },
     "dependencies": {
         "html-to-text": "^9.0.5",
-        "mailparser": "^3.7.4",
+        "mailparser": "^3.6.8",
         "node-imap": "^0.9.6",
         "nodemailer": "^7.0.6",
+        "pg": "^8.16.3",
+        "pg-large-object": "^2.0.0",
         "utf7": "^1.0.2"
     },
     "devDependencies": {
-        "@types/chai": "^5.2.2",
-        "@types/mailparser": "^3.4.6",
-        "@types/mocha": "^10.0.10",
-        "@types/node": "^24.5.2",
-        "@types/node-imap": "^0.9.3",
-        "@types/node-red": "^1.3.5",
-        "@types/node-red-node-test-helper": "^0.3.4",
-        "@types/nodemailer": "^7.0.1",
         "chai": "^4.3.4",
-        "cpx2": "^8.0.0",
         "mocha": "^11.7.2",
         "node-red": "^4.0.9",
-        "node-red-node-test-helper": "^0.3.5",
-        "npm-run-all": "^4.1.5",
-        "ts-node": "^10.9.2",
-        "typescript": "^5.9.2"
+        "node-red-node-test-helper": "^0.3.5"
     },
     "overrides": {
         "semver": ">=7.5.2",
         "axios": ">=1.12.0",
         "html-to-text": "^9.0.5",
+        "mailparser": "^3.6.8",
         "node-imap": "^0.9.6",
         "nodemailer": "^7.0.6",
         "utf7": "^1.0.2"

package/{src/html-to-text/html-to-text.html → processcube-html-to-text/processcube-html-to-text.html}

@@ -1,5 +1,5 @@
 <script type="text/javascript">
-    RED.nodes.registerType('html-to-text', {
+    RED.nodes.registerType('processcube-html-to-text', {
         category: 'ProcessCube Tools',
         color: '#02AFD6',
         defaults: {
@@ -9,12 +9,12 @@
         outputs: 1,
         icon: 'font-awesome/fa-sign-in',
         label: function () {
-            return this.name || 'html-to-text';
+            return this.name || 'processcube-html-to-text';
         },
     });
 </script>
 
-<script type="text/html" data-template-name="html-to-text">
+<script type="text/html" data-template-name="processcube-html-to-text">
     <div class="form-row">
         <label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
         <input type="text" id="node-input-name" placeholder="Name" />

package/processcube-html-to-text/processcube-html-to-text.js

@@ -0,0 +1,22 @@
+module.exports = function (RED) {
+    const { compile } = require('html-to-text');
+
+    function ProcesscubeHtmlToText(config) {
+        RED.nodes.createNode(this, config);
+        const node = this;
+
+        const options = {
+            wordwrap: 130,
+            // ...
+        };
+        const compiledConvert = compile(options); // options passed here
+
+        node.on('input', async function (msg) {
+            msg.payload = compiledConvert(msg.payload);
+
+            node.send(msg);
+        });
+    }
+
+    RED.nodes.registerType('processcube-html-to-text', ProcesscubeHtmlToText);
+};

package/storage/providers/fs.js

@@ -0,0 +1,117 @@
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const { pipeline } = require('stream');
+const { createHash } = require('crypto');
+const { promisify } = require('util');
+const pump = promisify(pipeline);
+
+class FsProvider {
+    constructor(opts = {}) {
+        this.baseDir = opts.baseDir || path.resolve(process.cwd(), 'data');
+    }
+
+    async init() {
+        await fsp.mkdir(this.baseDir, { recursive: true });
+    }
+
+    _buildPaths(id) {
+        const d = new Date();
+        const parts = [
+            String(d.getUTCFullYear()),
+            String(d.getUTCMonth() + 1).padStart(2, '0'),
+            String(d.getUTCDate()).padStart(2, '0'),
+        ];
+        const dir = path.join(this.baseDir, ...parts);
+        const filePath = path.join(dir, id);
+        const metaPath = path.join(dir, `${id}.json`);
+        return { dir, filePath, metaPath };
+    }
+
+    async store(readable, info) {
+        const { id, filename, contentType, metadata, createdAt } = info;
+        const { dir, filePath, metaPath } = this._buildPaths(id);
+        await fsp.mkdir(dir, { recursive: true });
+
+        const hash = createHash('sha256');
+        let size = 0;
+
+        const out = fs.createWriteStream(filePath);
+        readable.on('data', (chunk) => {
+            hash.update(chunk);
+            size += chunk.length;
+        });
+
+        await pump(readable, out);
+
+        const sha256 = hash.digest('hex');
+        const meta = { id, filename, contentType, size, sha256, metadata, createdAt };
+        await fsp.writeFile(metaPath, JSON.stringify(meta, null, 2));
+
+        return { size, sha256, path: filePath };
+    }
+
+    async get(id, options = { as: 'stream' }) {
+        // Find meta file by searching dated folders
+        const meta = await this._findMeta(id);
+        if (!meta) throw new Error(`File not found: ${id}`);
+        const filePath = meta.__filePath;
+
+        if (options.as === 'path') {
+            return { meta, payload: filePath };
+        }
+
+        if (options.as === 'buffer') {
+            const buf = await fsp.readFile(filePath);
+            return { meta, payload: buf };
+        }
+
+        // default: stream
+        const stream = fs.createReadStream(filePath);
+        return { meta, payload: stream };
+    }
+
+    async delete(id) {
+        const meta = await this._findMeta(id);
+        if (!meta) return; // idempotent
+        await fsp.unlink(meta.__filePath).catch(() => {});
+        await fsp.unlink(meta.__metaPath).catch(() => {});
+    }
+
+    async _findMeta(id) {
+        // Walk date folders (YYYY/MM/DD). For Performance: keep index/cache in prod.
+        const years = await this._ls(this.baseDir);
+        for (const y of years) {
+            const yearDir = path.join(this.baseDir, y);
+            const months = await this._ls(yearDir);
+            for (const m of months) {
+                const monthDir = path.join(yearDir, m);
+                const days = await this._ls(monthDir);
+                for (const d of days) {
+                    const dir = path.join(monthDir, d);
+                    const metaPath = path.join(dir, `${id}.json`);
+                    try {
+                        const raw = await fsp.readFile(metaPath, 'utf-8');
+                        const meta = JSON.parse(raw);
+                        meta.__metaPath = metaPath;
+                        meta.__filePath = path.join(dir, id);
+                        return meta;
+                    } catch (_) {
+                        /* continue */
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    async _ls(dir) {
+        try {
+            return (await fsp.readdir(dir)).filter((n) => !n.startsWith('.'));
+        } catch {
+            return [];
+        }
+    }
+}
+
+module.exports = FsProvider;