@5minds/node-red-contrib-processcube-tools 1.0.5 → 1.1.0-feature-6eab97-mg0ov11s

package/.env.template

@@ -2,3 +2,14 @@ EMAIL_SEND_PORT=
 EMAIL_SEND_HOST=
 EMAIL_SEND_USER=
 EMAIL_SEND_PASSWORD=
+
+EMAIL_RECEIVE_PORT=
+EMAIL_RECEIVE_HOST=
+EMAIL_RECEIVE_USER=
+EMAIL_RECEIVE_PASSWORD=
+
+PGUSER=postgres
+PGPASSWORD=postgres
+PGHOST=localhost
+PGPORT=5432
+PGDATABASE=enginedb

package/email-receiver/email-receiver.html

@@ -12,15 +12,28 @@
             tlsType: { value: 'bool' },
             user: { value: '', required: true, validate: RED.validators.typedInput('userType') },
             userType: { value: 'str' },
-            password: { value: '', required: true, type: 'password' },
+            password: { value: '', required: true },
             passwordType: { value: 'env', required: true },
             folder: { value: '', required: true, validate: RED.validators.typedInput('folderType') },
             folderType: { value: 'json' },
             markseen: { value: true, validate: RED.validators.typedInput('markseenType') },
             markseenType: { value: 'bool' },
+            sendstatus: { value: true },
         },
         inputs: 1,
-        outputs: 1,
+        // outputs: function () {
+        //     const sendstatus = this.sendstatus === true || this.sendstatus === 'true';
+        //     return sendstatus ? 2 : 1;
+        // },
+        outputs: 2,
+        outputLabels: function (index) {
+            if (index === 0) {
+                return 'Email Message';
+            }
+            if (index === 1) {
+                return 'Status';
+            }
+        },
         icon: 'font-awesome/fa-inbox',
         label: function () {
             return this.name || 'E-Mail Receiver';
@@ -118,6 +131,15 @@
         <input type="text" id="node-input-markseen" />
         <input type="hidden" id="node-input-markseenType" />
     </div>
+    <div class="form-row" style="margin-bottom: 3px;">
+        <input
+            type="checkbox"
+            checked
+            id="node-input-sendstatus"
+            style="display: inline-block; width: auto; vertical-align: top; margin-left: 5px;"
+        />
+        <label style="width:auto" for="node-input-sendstatus">Send status</label>
+    </div>
 </script>
 
 <script type="text/html" data-help-name="email-receiver">

package/email-receiver/email-receiver.js

@@ -24,6 +24,7 @@ module.exports = function (RED) {
             const imap_tls = RED.util.evaluateNodeProperty(config.tls, config.tlsType, node, msg);
             const imap_user = RED.util.evaluateNodeProperty(config.user, config.userType, node, msg);
             const imap_password = RED.util.evaluateNodeProperty(config.password, config.passwordType, node, msg);
+            const sendstatus = config.sendstatus === true || config.sendstatus === 'true';
 
             // Check if the folder is actually an array
             const imap_folder = RED.util.evaluateNodeProperty(config.folder, config.folderType, node, msg);
@@ -127,18 +128,51 @@ module.exports = function (RED) {
                     if (error) {
                         node.error('IMAP session terminated: ' + error.message);
                         node.status({ fill: 'red', shape: 'ring', text: 'connection error' });
+                        if (sendstatus) {
+                            node.send([null, { 
+                                payload: {
+                                    status: 'error',
+                                    message: error.message,
+                                    errors: state.errors,
+                                }
+                            }]);
+                        }
                     } else if (state.failures > 0) {
                         node.status({
                             fill: 'red',
                             shape: 'dot',
                             text: `Done, ${state.totalMails} mails from ${state.successes}/${state.totalFolders} folders. ${state.failures} failed.`,
                         });
+                        if (sendstatus) {
+                            node.send([null, { 
+                                payload: {
+                                    status: 'warning',
+                                    total: state.totalMails,
+                                    successes: state.successes,
+                                    failures: state.failures,
+                                    totalFolders: state.totalFolders,
+                                    errors: state.errors,
+                                }
+                            }]);
+                        }
+
                     } else {
                         node.status({
                             fill: 'green',
                             shape: 'dot',
                             text: `Done, fetched ${state.totalMails} mails from ${folders.join(', ')}.`,
                         });
+
+                        if (sendstatus) {
+                            node.send([null, { 
+                                payload: {
+                                    status: 'success',
+                                    total: state.totalMails,
+                                    folders: folders.join(', '),
+                                }
+                            }]);
+                        }
+
                     }
                     if (imap && imap.state !== 'disconnected') {
                         imap.end();

package/file-storage/file-storage.html

@@ -0,0 +1,97 @@
+<script type="text/javascript">
+    RED.nodes.registerType('file-storage', {
+        category: 'ProcessCube Tools',
+        color: '#02AFD6',
+        defaults: {
+            name: { value: '' },
+            provider: { value: 'fs' },
+            baseDir: { value: '' },
+            pgConnectionString: { value: '' },
+            pgSchema: { value: 'public' },
+            pgTable: { value: 'files' },
+            outputAs: { value: 'stream' },
+            defaultAction: { value: 'store' },
+        },
+        inputs: 1,
+        outputs: 1,
+        icon: 'file.png',
+        label: function () {
+            return this.name || 'file-storage';
+        },
+    });
+</script>
+
+<script type="text/html" data-template-name="file-storage">
+    <div class="form-row">
+        <label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
+        <input type="text" id="node-input-name" placeholder="file-storage" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-provider"><i class="fa fa-database"></i> Provider</label>
+        <select id="node-input-provider">
+            <option value="fs">Filesystem</option>
+            <option value="pg">PostgreSQL</option>
+        </select>
+    </div>
+    <div class="form-row">
+        <label for="node-input-outputAs"><i class="fa fa-share-square-o"></i> Output</label>
+        <select id="node-input-outputAs">
+            <option value="stream">Stream</option>
+            <option value="buffer">Buffer</option>
+            <option value="path">Path (nur FS)</option>
+        </select>
+    </div>
+    <hr />
+    <div class="form-tips">Filesystem</div>
+    <div class="form-row">
+        <label for="node-input-baseDir"><i class="fa fa-folder-open"></i> Base Dir</label>
+        <input type="text" id="node-input-baseDir" placeholder="/data/files" />
+    </div>
+    <hr />
+    <div class="form-tips">PostgreSQL</div>
+    <div class="form-row">
+        <label for="node-input-pgConnectionString"><i class="fa fa-plug"></i> Connection</label>
+        <input type="text" id="node-input-pgConnectionString" placeholder="postgres://user:pass@host:5432/db" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-pgSchema"><i class="fa fa-sitemap"></i> Schema</label>
+        <input type="text" id="node-input-pgSchema" placeholder="public" />
+    </div>
+    <div class="form-row">
+        <label for="node-input-pgTable"><i class="fa fa-table"></i> Tabelle</label>
+        <input type="text" id="node-input-pgTable" placeholder="files" />
+    </div>
+    <hr />
+    <div class="form-row">
+        <label for="node-input-defaultAction"><i class="fa fa-cog"></i> Default Action</label>
+        <select id="node-input-defaultAction">
+            <option value="store">store</option>
+            <option value="get">get</option>
+            <option value="delete">delete</option>
+        </select>
+    </div>
+</script>
+
+<script type="text/html" data-help-name="file-storage">
+    <p>
+        File-Storage-Node zum Speichern/Abrufen/Löschen von Dateien inkl. Metadaten. Provider: Filesystem (Datei + JSON)
+        oder PostgreSQL (Large Objects + Metadaten-Tabelle).
+    </p>
+    <h3>Input</h3>
+    <pre>
+  msg.action = "store" | "get" | "delete"
+  msg.payload = Buffer | Readable | String (bei store)
+  msg.file = {
+    id?: string (bei get/delete),
+    filename?: string,
+    contentType?: string,
+    metadata?: object
+  }
+  </pre
+    >
+    <h3>Output</h3>
+    <p>
+        Bei <code>store</code>: <code>msg.payload</code> enthält Metadaten inkl. <code>id</code>. Bei <code>get</code>:
+        <code>msg.payload</code> ist Stream/Buffer/Pfad (je nach Option), Metadaten in <code>msg.file</code>.
+    </p>
+</script>

package/file-storage/file-storage.js

@@ -0,0 +1,70 @@
+module.exports = function (RED) {
+    const StorageCore = require('../storage/storage-core');
+
+    function FileStorageNode(config) {
+        RED.nodes.createNode(this, config);
+        const node = this;
+
+        // Node-Konfiguration
+        node.provider = config.provider || 'fs';
+        node.baseDir = config.baseDir;
+        node.pg = {
+            connectionString: config.pgConnectionString,
+            schema: config.pgSchema || 'public',
+            table: config.pgTable || 'files',
+        };
+        node.outputAs = config.outputAs || 'stream'; // 'stream' | 'buffer' | 'path' (path nur fs)
+
+        // Storage-Kern
+        const storage = new StorageCore({
+            provider: node.provider,
+            fs: { baseDir: node.baseDir },
+            pg: node.pg,
+        });
+
+        storage.init().catch((err) => node.error(err));
+
+        node.on('input', async function (msg, send, done) {
+            try {
+                const action = msg.action || config.defaultAction || 'store';
+                if (action === 'store') {
+                    const file = msg.file || {};
+                    const result = await storage.store(msg.payload, file);
+                    msg.payload = result;
+                    msg.file = { ...file, ...result };
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                if (action === 'get') {
+                    const id = msg.file && msg.file.id;
+                    if (!id) throw new Error('file.id is required for get');
+                    const { meta, payload } = await storage.get(id, { as: node.outputAs });
+                    msg.file = { ...meta, id: meta.id };
+                    msg.payload = payload;
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                if (action === 'delete') {
+                    const id = msg.file && msg.file.id;
+                    if (!id) throw new Error('file.id is required for delete');
+                    const result = await storage.delete(id);
+                    msg.payload = result;
+                    send(msg);
+                    done();
+                    return;
+                }
+
+                throw new Error(`Unknown action: ${action}`);
+            } catch (err) {
+                node.error(err, msg);
+                if (done) done(err);
+            }
+        });
+    }
+
+    RED.nodes.registerType('file-storage', FileStorageNode);
+};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@5minds/node-red-contrib-processcube-tools",
-    "version": "1.0.5",
+    "version": "1.1.0-feature-6eab97-mg0ov11s",
     "license": "MIT",
     "description": "Node-RED tools nodes for ProcessCube",
     "scripts": {
@@ -19,6 +19,10 @@
         {
             "name": "Diana Stefan",
             "email": "Diana.Stefan@5Minds.de"
+        },
+        {
+            "name": "Thorsten Kallweit",
+            "email": "Thorsten.Kallweit@5Minds.de"
         }
     ],
     "repository": {
@@ -38,7 +42,8 @@
         "nodes": {
             "EmailReceiver": "email-receiver/email-receiver.js",
             "EmailSender": "email-sender/email-sender.js",
-            "HtmlToText": "processcube-html-to-text/processcube-html-to-text.js"
+            "HtmlToText": "processcube-html-to-text/processcube-html-to-text.js",
+            "FileStorage": "file-storage/file-storage.js"
         },
         "examples": "examples"
     },
@@ -47,6 +52,8 @@
         "mailparser": "^3.6.8",
         "node-imap": "^0.9.6",
         "nodemailer": "^7.0.6",
+        "pg": "^8.16.3",
+        "pg-large-object": "^2.0.0",
         "utf7": "^1.0.2"
     },
     "devDependencies": {

package/storage/providers/fs.js

@@ -0,0 +1,117 @@
+const fs = require('fs');
+const fsp = require('fs/promises');
+const path = require('path');
+const { pipeline } = require('stream');
+const { createHash } = require('crypto');
+const { promisify } = require('util');
+const pump = promisify(pipeline);
+
+class FsProvider {
+    constructor(opts = {}) {
+        this.baseDir = opts.baseDir || path.resolve(process.cwd(), 'data');
+    }
+
+    async init() {
+        await fsp.mkdir(this.baseDir, { recursive: true });
+    }
+
+    _buildPaths(id) {
+        const d = new Date();
+        const parts = [
+            String(d.getUTCFullYear()),
+            String(d.getUTCMonth() + 1).padStart(2, '0'),
+            String(d.getUTCDate()).padStart(2, '0'),
+        ];
+        const dir = path.join(this.baseDir, ...parts);
+        const filePath = path.join(dir, id);
+        const metaPath = path.join(dir, `${id}.json`);
+        return { dir, filePath, metaPath };
+    }
+
+    async store(readable, info) {
+        const { id, filename, contentType, metadata, createdAt } = info;
+        const { dir, filePath, metaPath } = this._buildPaths(id);
+        await fsp.mkdir(dir, { recursive: true });
+
+        const hash = createHash('sha256');
+        let size = 0;
+
+        const out = fs.createWriteStream(filePath);
+        readable.on('data', (chunk) => {
+            hash.update(chunk);
+            size += chunk.length;
+        });
+
+        await pump(readable, out);
+
+        const sha256 = hash.digest('hex');
+        const meta = { id, filename, contentType, size, sha256, metadata, createdAt };
+        await fsp.writeFile(metaPath, JSON.stringify(meta, null, 2));
+
+        return { size, sha256, path: filePath };
+    }
+
+    async get(id, options = { as: 'stream' }) {
+        // Find meta file by searching dated folders
+        const meta = await this._findMeta(id);
+        if (!meta) throw new Error(`File not found: ${id}`);
+        const filePath = meta.__filePath;
+
+        if (options.as === 'path') {
+            return { meta, payload: filePath };
+        }
+
+        if (options.as === 'buffer') {
+            const buf = await fsp.readFile(filePath);
+            return { meta, payload: buf };
+        }
+
+        // default: stream
+        const stream = fs.createReadStream(filePath);
+        return { meta, payload: stream };
+    }
+
+    async delete(id) {
+        const meta = await this._findMeta(id);
+        if (!meta) return; // idempotent
+        await fsp.unlink(meta.__filePath).catch(() => {});
+        await fsp.unlink(meta.__metaPath).catch(() => {});
+    }
+
+    async _findMeta(id) {
+        // Walk date folders (YYYY/MM/DD). For Performance: keep index/cache in prod.
+        const years = await this._ls(this.baseDir);
+        for (const y of years) {
+            const yearDir = path.join(this.baseDir, y);
+            const months = await this._ls(yearDir);
+            for (const m of months) {
+                const monthDir = path.join(yearDir, m);
+                const days = await this._ls(monthDir);
+                for (const d of days) {
+                    const dir = path.join(monthDir, d);
+                    const metaPath = path.join(dir, `${id}.json`);
+                    try {
+                        const raw = await fsp.readFile(metaPath, 'utf-8');
+                        const meta = JSON.parse(raw);
+                        meta.__metaPath = metaPath;
+                        meta.__filePath = path.join(dir, id);
+                        return meta;
+                    } catch (_) {
+                        /* continue */
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    async _ls(dir) {
+        try {
+            return (await fsp.readdir(dir)).filter((n) => !n.startsWith('.'));
+        } catch {
+            return [];
+        }
+    }
+}
+
+module.exports = FsProvider;

package/storage/providers/postgres.js

@@ -0,0 +1,159 @@
+const { Pool } = require('pg');
+const { LargeObjectManager } = require('pg-large-object');
+const { pipeline, PassThrough } = require('stream');
+const { createHash } = require('crypto');
+const { promisify } = require('util');
+const pump = promisify(pipeline);
+
+class PgProvider {
+    constructor(opts = {}) {
+        // this.connectionString = opts.connectionString || process.env.PG_URL || 'postgres://localhost/postgres';
+        this.schema = opts.schema || 'public';
+        this.table = opts.table || 'files';
+        // this.pool = new Pool({ connectionString: this.connectionString });
+        this.pool = new Pool();
+    }
+
+    async init() {
+        const client = await this.pool.connect();
+        try {
+            await client.query(`CREATE TABLE IF NOT EXISTS ${this.schema}.${this.table} (
+        id UUID PRIMARY KEY,
+        loid OID NOT NULL,
+        filename TEXT,
+        content_type TEXT,
+        size BIGINT,
+        sha256 TEXT,
+        metadata JSONB,
+        created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+      )`);
+            await client.query(
+                `CREATE INDEX IF NOT EXISTS idx_${this.table}_created_at ON ${this.schema}.${this.table}(created_at)`,
+            );
+        } finally {
+            client.release();
+        }
+    }
+
+    async store(readable, info) {
+    const { id, filename, contentType, metadata, createdAt } = info;
+    const client = await this.pool.connect();
+    try {
+        const lom = new LargeObjectManager({pg: client});
+        await client.query('BEGIN');
+        const bufSize = 16384;
+        
+        const result = await lom.createAndWritableStreamAsync(bufSize);
+        const oid =result[0];
+        const stream = result[1];
+        if (!oid || !stream) {
+            throw new Error('Failed to create large object');
+        }
+        const hash = createHash('sha256');
+        let size = 0;
+
+        // Wir berechnen Hash und Größe "on the fly", während wir in das Large Object schreiben
+        const hashAndSize = new PassThrough();
+        hashAndSize.on('data', (chunk) => {
+            hash.update(chunk);
+            size += chunk.length;
+        });
+
+        // readable -> hashAndSize -> stream
+        const pipelinePromise = pump(readable, hashAndSize, stream);
+
+        await pipelinePromise;
+
+        const sha256 = hash.digest('hex');
+
+        await client.query(
+            `INSERT INTO ${this.schema}.${this.table} (id, loid, filename, content_type, size, sha256, metadata, created_at)
+             VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)`,
+            [id, oid, filename, contentType, size, sha256, JSON.stringify(metadata || {}), createdAt],
+        );
+
+        await client.query('COMMIT');
+        return { size, sha256, oid };
+    } catch (err) {
+        await client.query('ROLLBACK').catch(() => {});
+        throw err;
+    } finally {
+        client.release();
+    }
+}
+
+    async get(id, options = { as: 'stream' }) {
+        const client = await this.pool.connect();
+        try {
+            const { rows } = await client.query(`SELECT * FROM ${this.schema}.${this.table} WHERE id=$1`, [id]);
+            if (rows.length === 0) throw new Error(`File not found: ${id}`);
+            const meta = rows[0];
+
+            const bufSize = 16384;
+            if (options.as === 'buffer') {
+                // Stream LO into memory
+                await client.query('BEGIN');
+                const lom = new LargeObjectManager({ pg: client });
+                
+                const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
+                const totalSize = ro[0];
+                const stream = ro[1];
+                if (!stream) {
+                    throw new Error('Failed to open large object for reading');
+                }   
+                const chunks = [];
+                stream.on('data', (c) => chunks.push(c));
+                await new Promise((res, rej) => stream.on('end', res).on('error', rej));
+                await client.query('COMMIT');
+                client.release();
+                return { meta, payload: Buffer.concat(chunks) };
+            }
+
+            if (options.as === 'path') {
+                throw new Error('options.as="path" is not supported by Postgres provider');
+            }
+
+            // default: stream – wrap LO stream so we can close txn when done
+            await client.query('BEGIN');
+            const lom = new LargeObjectManager({ pg: client });
+            const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
+            const totalSize = ro[0];
+            const stream = ro[1];
+            const pass = new PassThrough();
+            stream.pipe(pass);
+            const done = new Promise((res, rej) => pass.on('end', res).on('error', rej));
+            done.finally(async () => {
+                await client.query('COMMIT').catch(() => {});
+                client.release();
+            });
+            
+            // Do not release here; we release in finally of wrapper. We return early, so prevent double release.
+            return { meta, payload: pass };
+        } catch (err) {
+            
+            throw err;
+        }
+    }
+
+    async delete(id) {
+        const client = await this.pool.connect();
+        try {
+            await client.query('BEGIN');
+            const { rows } = await client.query(`DELETE FROM ${this.schema}.${this.table} WHERE id=$1 RETURNING loid`, [
+                id,
+            ]);
+            if (rows.length) {
+                const lom = new LargeObjectManager({ pg: client });
+                await lom.unlinkAsync(rows[0].loid);
+            }
+            await client.query('COMMIT');
+        } catch (err) {
+            await client.query('ROLLBACK').catch(() => {});
+            throw err;
+        } finally {
+            client.release();
+        }
+    }
+}
+
+module.exports = PgProvider;

package/storage/storage-core.js

@@ -0,0 +1,73 @@
+const { v4: uuidv4 } = require('uuid');
+const { Readable } = require('stream');
+const FsProvider = require('./providers/fs');
+const PgProvider = require('./providers/postgres');
+
+function ensureReadable(payload) {
+    if (!payload && payload !== 0) {
+        throw new Error('No payload provided for storage');
+    }
+    if (Buffer.isBuffer(payload) || typeof payload === 'string' || typeof payload === 'number') {
+        return Readable.from(Buffer.isBuffer(payload) ? payload : Buffer.from(String(payload)));
+    }
+    if (payload && typeof payload.pipe === 'function') {
+        return payload; // Readable stream
+    }
+    throw new Error('Unsupported payload type. Use Buffer, string, number, or Readable stream.');
+}
+
+class StorageCore {
+    /**
+     * @param {Object} config
+     * @param {('fs'|'pg')} config.provider
+     * @param {Object} [config.fs]
+     * @param {string} [config.fs.baseDir]
+     * @param {Object} [config.pg]
+     * @param {string} [config.pg.connectionString]
+     * @param {string} [config.pg.schema]
+     * @param {string} [config.pg.table]
+     */
+    constructor(config = {}) {
+        this.config = config;
+        const p = config.provider || 'fs';
+        if (p === 'fs') this.provider = new FsProvider(config.fs || {});
+        else if (p === 'pg') this.provider = new PgProvider(config.pg || {});
+        else throw new Error(`Unknown provider: ${p}`);
+    }
+
+    async init() {
+        if (typeof this.provider.init === 'function') {
+            await this.provider.init();
+        }
+    }
+
+    /** Store a file */
+    async store(payload, file = {}) {
+        const stream = ensureReadable(payload);
+        const id = uuidv4();
+        const info = {
+            id,
+            filename: file.filename || id,
+            contentType: file.contentType || 'application/octet-stream',
+            metadata: file.metadata || {},
+            createdAt: new Date().toISOString(),
+        };
+        const result = await this.provider.store(stream, info);
+        return { ...info, ...result, storage: this.config.provider || 'fs' };
+    }
+
+    /** Get a file by id */
+    async get(id, options = { as: 'stream' }) {
+        if (!id) throw new Error('id is required');
+        return this.provider.get(id, options);
+    }
+
+    /** Delete a file by id */
+    async delete(id) {
+        if (!id) throw new Error('id is required');
+        await this.provider.delete(id);
+        return { id, deleted: true };
+    }
+}
+
+module.exports = StorageCore;