@5minds/node-red-contrib-processcube-tools 1.2.0-feature-608421-mg9cjskq → 1.2.0-feature-6d921c-mgp4itmd

package/file-storage/file-storage.html

@@ -1,203 +0,0 @@
-<script type="text/javascript">
-    RED.nodes.registerType('file-storage', {
-        category: 'ProcessCube Tools',
-        color: '#02AFD6',
-        defaults: {
-            name: { value: '' },
-            provider: { value: 'fs' },
-            baseDir: { value: '' },
-            usernameType: { value: 'str' },
-            username: { value: '', required: true, validate: RED.validators.typedInput('usernameType') },
-            passwordType: { value: 'str' },
-            password: { value: '', required: true, validate: RED.validators.typedInput('passwordType') },
-            hostType: { value: 'str' },
-            host: { value: '', required: true, validate: RED.validators.typedInput('hostType') },
-            portType: { value: 'str' },
-            port: { value: '', required: true, validate: RED.validators.typedInput('portType') },
-            databaseType: { value: 'str' },
-            database: { value: '', required: true, validate: RED.validators.typedInput('databaseType') },
-            
-            pgSchema: { value: 'public' },
-            pgTable: { value: 'files' },
-            outputAs: { value: 'stream' },
-            defaultAction: { value: 'store' },
-        },
-        inputs: 1,
-        outputs: 1,
-        icon: 'file.png',
-        label: function () {
-            return this.name || 'file-storage';
-        },
-        oneditprepare: function () {
-            // postgres fields user, password, host, port, database
-            $('#node-input-username').typedInput({
-                default: 'str',
-                types: ['str', 'env'],
-                typeField: '#node-input-usernameType',
-            });
-            $('#node-input-password').typedInput({
-                default: 'str',
-                types: ['str', 'env'],
-                typeField: '#node-input-passwordType',
-            });
-            $('#node-input-host').typedInput({
-                default: 'str',
-                types: ['str', 'env'],
-                typeField: '#node-input-hostType',
-            });
-            $('#node-input-port').typedInput({
-                default: 'str',
-                types: ['str', 'env'],
-                typeField: '#node-input-portType',
-            });
-            $('#node-input-database').typedInput({
-                default: 'str',
-                types: ['str', 'env'],
-                typeField: '#node-input-databaseType',
-            });
-        },
-    });
-</script>
-
-<script type="text/html" data-template-name="file-storage">
-    <div class="form-row">
-        <label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
-        <input type="text" id="node-input-name" placeholder="file-storage" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-provider"><i class="fa fa-database"></i> Provider</label>
-        <select id="node-input-provider">
-            <option value="fs">Filesystem</option>
-            <option value="pg">PostgreSQL</option>
-        </select>
-    </div>
-    <div class="form-row">
-        <label for="node-input-outputAs"><i class="fa fa-share-square-o"></i> Output</label>
-        <select id="node-input-outputAs">
-            <option value="stream">Stream</option>
-            <option value="buffer">Buffer</option>
-            <option value="path">Path (nur FS)</option>
-        </select>
-    </div>
-    <hr />
-    <div class="form-tips">Filesystem</div>
-    <div class="form-row">
-        <label for="node-input-baseDir"><i class="fa fa-folder-open"></i> Base Dir</label>
-        <input type="text" id="node-input-baseDir" placeholder="/data/files" />
-    </div>
-    <hr />
-    <div class="form-tips">PostgreSQL</div>
-    <div class="form-row">
-        <label for="node-input-username"><i class="fa fa-user"></i> Username</label>
-        <input type="text" id="node-input-username" placeholder="postgres" />
-        <input type="hidden" id="node-input-usernameType" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-password"><i class="fa fa-key"></i> Password</label>
-        <input type="text" id="node-input-password" placeholder="postgres" />
-         <input type="hidden" id="node-input-passwordType" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-host"><i class="fa fa-server"></i> Host</label>
-        <input type="text" id="node-input-host" placeholder="localhost" />
-         <input type="hidden" id="node-input-hostType" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-port"><i class="fa fa-plug"></i> Port</label>
-        <input type="text" id="node-input-port" placeholder="5432" />
-         <input type="hidden" id="node-input-portType" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-database"><i class="fa fa-database"></i> Database</label>
-        <input type="text" id="node-input-database" placeholder="postgres" />
-         <input type="hidden" id="node-input-databaseType" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-pgSchema"><i class="fa fa-sitemap"></i> Schema</label>
-        <input type="text" id="node-input-pgSchema" placeholder="public" />
-    </div>
-    <div class="form-row">
-        <label for="node-input-pgTable"><i class="fa fa-table"></i> Table</label>
-        <input type="text" id="node-input-pgTable" placeholder="files" />
-    </div>
-    <hr />
-    <div class="form-row">
-        <label for="node-input-defaultAction"><i class="fa fa-cog"></i> Default Action</label>
-        <select id="node-input-defaultAction">
-            <option value="store">store</option>
-            <option value="get">get</option>
-            <option value="delete">delete</option>
-        </select>
-    </div>
-</script>
-
-<script type="text/html" data-help-name="file-storage">
-    <h2>File Storage Node</h2>
-    <p>
-        A Node-RED node for storing, retrieving, and deleting files (including metadata) using either the local filesystem or PostgreSQL (Large Objects + metadata table) as a backend.
-    </p>
-    <h3>Features</h3>
-    <ul>
-        <li><b>Providers:</b>
-            <ul>
-                <li>Filesystem (stores file and metadata as JSON)</li>
-                <li>PostgreSQL (stores file as Large Object and metadata in a table)</li>
-            </ul>
-        </li>
-        <li><b>Actions:</b> Store, Retrieve, Delete files</li>
-        <li><b>Flexible output:</b> Stream, Buffer, or Path (filesystem only)</li>
-    </ul>
-    <h3>Node Properties</h3>
-    <ul>
-        <li><b>Name:</b> Optional node label.</li>
-        <li><b>Provider:</b> Select between Filesystem and PostgreSQL.</li>
-        <li><b>Output:</b> Choose the output type for retrieval: Stream, Buffer, or Path (Path only for filesystem).</li>
-        <li><b>Base Dir:</b> (Filesystem) Directory where files are stored.</li>
-        <li><b>Schema:</b> (PostgreSQL) Database schema (default: public).</li>
-        <li><b>Table:</b> (PostgreSQL) Table for metadata (default: files).</li>
-        <li><b>Default Action:</b> Default action if not specified in the message (store, get, or delete).</li>
-    </ul>
-    <h3>Input</h3>
-    <pre>
-msg.action = "store" | "get" | "delete" // Optional, overrides defaultAction
-msg.payload = Buffer | ReadableStream | String // For "store"
-msg.file = {
-  id?: string,         // For "get" or "delete"
-  filename?: string,   // For "store"
-  contentType?: string,// For "store"
-  metadata?: object    // For "store"
-}
-    </pre>
-    <h3>Output</h3>
-    <ul>
-        <li><b>store:</b> <code>msg.payload</code> contains metadata including the generated <code>id</code>. <code>msg.file</code> contains the merged file info and result.</li>
-        <li><b>get:</b> <code>msg.payload</code> contains the file as a Stream, Buffer, or Path (depending on output setting). <code>msg.file</code> contains the file metadata.</li>
-        <li><b>delete:</b> <code>msg.payload</code> contains the result of the delete operation.</li>
-    </ul>
-    <h3>Example Usage</h3>
-    <pre>
-// Store a file:
-msg.action = "store";
-msg.payload = Buffer.from("Hello World");
-msg.file = {
-  filename: "hello.txt",
-  contentType: "text/plain",
-  metadata: { author: "Alice" }
-};
-
-// Retrieve a file:
-msg.action = "get";
-msg.file = { id: "your-file-id" };
-
-// Delete a file:
-msg.action = "delete";
-msg.file = { id: "your-file-id" };
-    </pre>
-    <h3>Notes</h3>
-    <ul>
-        <li>For PostgreSQL, ensure the connection string, schema, and table exist and the user has the necessary permissions.</li>
-        <li>For filesystem storage, ensure the base directory is writable by Node-RED.</li>
-        <li>The node is designed to handle large files efficiently using streams.</li>
-    </ul>
-    <p><b>Enjoy using the File Storage Node in your Node-RED flows!</b></p>
-</script>

package/file-storage/file-storage.js

@@ -1,148 +0,0 @@
-/**
- * # File Storage Node
- *
- * A Node-RED node for storing, retrieving, and deleting files (including metadata) using either the local filesystem or PostgreSQL (Large Objects + metadata table) as a backend.
- *
- * ## Features
- * - **Providers:**
- *   - Filesystem (stores file and metadata as JSON)
- *   - PostgreSQL (stores file as Large Object and metadata in a table)
- * - **Actions:**
- *   - Store a file
- *   - Retrieve a file
- *   - Delete a file
- * - **Flexible output:**
- *   - Stream, Buffer, or Path (filesystem only)
- *
- * ## Node Properties
- * - **Name:** Optional node label.
- * - **Provider:** Select between `Filesystem` and `PostgreSQL`.
- * - **Output:** Choose the output type for retrieval: `Stream`, `Buffer`, or `Path` (Path only for filesystem).
- * - **Base Dir:** (Filesystem) Directory where files are stored.
- * - **Connection:** (PostgreSQL) Connection string for the database.
- * - **Schema:** (PostgreSQL) Database schema (default: `public`).
- * - **Table:** (PostgreSQL) Table for metadata (default: `files`).
- * - **Default Action:** Default action if not specified in the message (`store`, `get`, or `delete`).
- *
- * ## Input
- * The node expects the following properties in the incoming message:
- * ```
- * msg.action = "store" | "get" | "delete" // Optional, overrides defaultAction
- * msg.payload = Buffer | ReadableStream | String // For "store"
- * msg.file = {
- *   id?: string,         // For "get" or "delete"
- *   filename?: string,   // For "store"
- *   contentType?: string,// For "store"
- *   metadata?: object    // For "store"
- * }
- * ```
- *
- * ## Output
- * - For **store**:
- *   - `msg.payload` contains metadata including the generated `id`.
- *   - `msg.file` contains the merged file info and result.
- * - For **get**:
- *   - `msg.payload` contains the file as a Stream, Buffer, or Path (depending on output setting).
- *   - `msg.file` contains the file metadata.
- * - For **delete**:
- *   - `msg.payload` contains the result of the delete operation.
- *
- * ## Example Usage
- * // Store a file:
- * msg.action = "store";
- * msg.payload = Buffer.from("Hello World");
- * msg.file = {
- *   filename: "hello.txt",
- *   contentType: "text/plain",
- *   metadata: { author: "Alice" }
- * };
- *
- * // Retrieve a file:
- * msg.action = "get";
- * msg.file = { id: "your-file-id" };
- *
- * // Delete a file:
- * msg.action = "delete";
- * msg.file = { id: "your-file-id" };
- *
- * ## Notes
- * - For PostgreSQL, ensure the connection string, schema, and table exist and the user has the necessary permissions.
- * - For filesystem storage, ensure the base directory is writable by Node-RED.
- * - The node is designed to handle large files efficiently using streams.
- *
- * Enjoy using the File Storage Node in your Node-RED flows!
- */
-module.exports = function (RED) {
-    const StorageCore = require('../storage/storage-core');
-
-    function FileStorageNode(config) {
-        RED.nodes.createNode(this, config);
-        const node = this;
-
-        // Node-Konfiguration
-        node.provider = config.provider || 'fs';
-        node.baseDir = config.baseDir;
-        node.pg = {
-            username:  RED.util.evaluateNodeProperty(config.username, config.usernameType, node) || 'postgres',
-            password: RED.util.evaluateNodeProperty(config.password, config.passwordType, node) || 'postgres',
-            host: RED.util.evaluateNodeProperty(config.host, config.hostType, node) || 'localhost',
-            port: RED.util.evaluateNodeProperty(config.port, config.portType, node) || 5432,
-            database: RED.util.evaluateNodeProperty(config.database, config.databaseType, node) || 'postgres',
-            schema: config.pgSchema || 'public',
-            table: config.pgTable || 'files',
-        };
-        node.outputAs = config.outputAs || 'stream'; // 'stream' | 'buffer' | 'path' (path nur fs)
-
-        // Storage-Kern
-        const storage = new StorageCore({
-            provider: node.provider,
-            fs: { baseDir: node.baseDir },
-            pg: node.pg,
-        });
-
-        storage.init().catch((err) => node.error(err));
-
-        node.on('input', async function (msg, send, done) {
-            try {
-                const action = msg.action || config.defaultAction || 'store';
-                if (action === 'store') {
-                    const file = msg.file || {};
-                    const result = await storage.store(msg.payload, file);
-                    msg.payload = result;
-                    msg.file = { ...file, ...result };
-                    send(msg);
-                    done();
-                    return;
-                }
-
-                if (action === 'get') {
-                    const id = msg.file && msg.file.id;
-                    if (!id) throw new Error('file.id is required for get');
-                    const { meta, payload } = await storage.get(id, { as: node.outputAs });
-                    msg.file = { ...meta, id: meta.id };
-                    msg.payload = payload;
-                    send(msg);
-                    done();
-                    return;
-                }
-
-                if (action === 'delete') {
-                    const id = msg.file && msg.file.id;
-                    if (!id) throw new Error('file.id is required for delete');
-                    const result = await storage.delete(id);
-                    msg.payload = result;
-                    send(msg);
-                    done();
-                    return;
-                }
-
-                throw new Error(`Unknown action: ${action}`);
-            } catch (err) {
-                node.error(err, msg);
-                if (done) done(err);
-            }
-        });
-    }
-
-    RED.nodes.registerType('file-storage', FileStorageNode);
-};

package/processcube-html-to-text/processcube-html-to-text.js

@@ -1,22 +0,0 @@
-module.exports = function (RED) {
-    const { compile } = require('html-to-text');
-
-    function ProcesscubeHtmlToText(config) {
-        RED.nodes.createNode(this, config);
-        const node = this;
-
-        const options = {
-            wordwrap: 130,
-            // ...
-        };
-        const compiledConvert = compile(options); // options passed here
-
-        node.on('input', async function (msg) {
-            msg.payload = compiledConvert(msg.payload);
-
-            node.send(msg);
-        });
-    }
-
-    RED.nodes.registerType('processcube-html-to-text', ProcesscubeHtmlToText);
-};

package/storage/providers/fs.js

@@ -1,117 +0,0 @@
-const fs = require('fs');
-const fsp = require('fs/promises');
-const path = require('path');
-const { pipeline } = require('stream');
-const { createHash } = require('crypto');
-const { promisify } = require('util');
-const pump = promisify(pipeline);
-
-class FsProvider {
-    constructor(opts = {}) {
-        this.baseDir = opts.baseDir || path.resolve(process.cwd(), 'data');
-    }
-
-    async init() {
-        await fsp.mkdir(this.baseDir, { recursive: true });
-    }
-
-    _buildPaths(id) {
-        const d = new Date();
-        const parts = [
-            String(d.getUTCFullYear()),
-            String(d.getUTCMonth() + 1).padStart(2, '0'),
-            String(d.getUTCDate()).padStart(2, '0'),
-        ];
-        const dir = path.join(this.baseDir, ...parts);
-        const filePath = path.join(dir, id);
-        const metaPath = path.join(dir, `${id}.json`);
-        return { dir, filePath, metaPath };
-    }
-
-    async store(readable, info) {
-        const { id, filename, contentType, metadata, createdAt } = info;
-        const { dir, filePath, metaPath } = this._buildPaths(id);
-        await fsp.mkdir(dir, { recursive: true });
-
-        const hash = createHash('sha256');
-        let size = 0;
-
-        const out = fs.createWriteStream(filePath);
-        readable.on('data', (chunk) => {
-            hash.update(chunk);
-            size += chunk.length;
-        });
-
-        await pump(readable, out);
-
-        const sha256 = hash.digest('hex');
-        const meta = { id, filename, contentType, size, sha256, metadata, createdAt };
-        await fsp.writeFile(metaPath, JSON.stringify(meta, null, 2));
-
-        return { size, sha256, path: filePath };
-    }
-
-    async get(id, options = { as: 'stream' }) {
-        // Find meta file by searching dated folders
-        const meta = await this._findMeta(id);
-        if (!meta) throw new Error(`File not found: ${id}`);
-        const filePath = meta.__filePath;
-
-        if (options.as === 'path') {
-            return { meta, payload: filePath };
-        }
-
-        if (options.as === 'buffer') {
-            const buf = await fsp.readFile(filePath);
-            return { meta, payload: buf };
-        }
-
-        // default: stream
-        const stream = fs.createReadStream(filePath);
-        return { meta, payload: stream };
-    }
-
-    async delete(id) {
-        const meta = await this._findMeta(id);
-        if (!meta) return; // idempotent
-        await fsp.unlink(meta.__filePath).catch(() => {});
-        await fsp.unlink(meta.__metaPath).catch(() => {});
-    }
-
-    async _findMeta(id) {
-        // Walk date folders (YYYY/MM/DD). For Performance: keep index/cache in prod.
-        const years = await this._ls(this.baseDir);
-        for (const y of years) {
-            const yearDir = path.join(this.baseDir, y);
-            const months = await this._ls(yearDir);
-            for (const m of months) {
-                const monthDir = path.join(yearDir, m);
-                const days = await this._ls(monthDir);
-                for (const d of days) {
-                    const dir = path.join(monthDir, d);
-                    const metaPath = path.join(dir, `${id}.json`);
-                    try {
-                        const raw = await fsp.readFile(metaPath, 'utf-8');
-                        const meta = JSON.parse(raw);
-                        meta.__metaPath = metaPath;
-                        meta.__filePath = path.join(dir, id);
-                        return meta;
-                    } catch (_) {
-                        /* continue */
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    async _ls(dir) {
-        try {
-            return (await fsp.readdir(dir)).filter((n) => !n.startsWith('.'));
-        } catch {
-            return [];
-        }
-    }
-}
-
-module.exports = FsProvider;

package/storage/providers/postgres.js

@@ -1,160 +0,0 @@
-const { Pool } = require('pg');
-const { LargeObjectManager } = require('pg-large-object');
-const { pipeline, PassThrough } = require('stream');
-const { createHash } = require('crypto');
-const { promisify } = require('util');
-const pump = promisify(pipeline);
-
-class PgProvider {
-    constructor(opts = {}) {
-        const conString = "postgres://" + opts.username + ":" + opts.password + "@" +  opts.host + ":" + opts.port + "/" + opts.database;
-        this.connectionString = conString
-        this.schema = opts.schema || 'public';
-        this.table = opts.table || 'files';
-        this.pool = new Pool({ connectionString: this.connectionString });
-        this.pool = new Pool();
-    }
-
-    async init() {
-        const client = await this.pool.connect();
-        try {
-            await client.query(`CREATE TABLE IF NOT EXISTS ${this.schema}.${this.table} (
-        id UUID PRIMARY KEY,
-        loid OID NOT NULL,
-        filename TEXT,
-        content_type TEXT,
-        size BIGINT,
-        sha256 TEXT,
-        metadata JSONB,
-        created_at TIMESTAMPTZ NOT NULL DEFAULT now()
-      )`);
-            await client.query(
-                `CREATE INDEX IF NOT EXISTS idx_${this.table}_created_at ON ${this.schema}.${this.table}(created_at)`,
-            );
-        } finally {
-            client.release();
-        }
-    }
-
-    async store(readable, info) {
-    const { id, filename, contentType, metadata, createdAt } = info;
-    const client = await this.pool.connect();
-    try {
-        const lom = new LargeObjectManager({pg: client});
-        await client.query('BEGIN');
-        const bufSize = 16384;
-        
-        const result = await lom.createAndWritableStreamAsync(bufSize);
-        const oid =result[0];
-        const stream = result[1];
-        if (!oid || !stream) {
-            throw new Error('Failed to create large object');
-        }
-        const hash = createHash('sha256');
-        let size = 0;
-
-        // Wir berechnen Hash und Größe "on the fly", während wir in das Large Object schreiben
-        const hashAndSize = new PassThrough();
-        hashAndSize.on('data', (chunk) => {
-            hash.update(chunk);
-            size += chunk.length;
-        });
-
-        // readable -> hashAndSize -> stream
-        const pipelinePromise = pump(readable, hashAndSize, stream);
-
-        await pipelinePromise;
-
-        const sha256 = hash.digest('hex');
-
-        await client.query(
-            `INSERT INTO ${this.schema}.${this.table} (id, loid, filename, content_type, size, sha256, metadata, created_at)
-             VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)`,
-            [id, oid, filename, contentType, size, sha256, JSON.stringify(metadata || {}), createdAt],
-        );
-
-        await client.query('COMMIT');
-        return { size, sha256, oid };
-    } catch (err) {
-        await client.query('ROLLBACK').catch(() => {});
-        throw err;
-    } finally {
-        client.release();
-    }
-}
-
-    async get(id, options = { as: 'stream' }) {
-        const client = await this.pool.connect();
-        try {
-            const { rows } = await client.query(`SELECT * FROM ${this.schema}.${this.table} WHERE id=$1`, [id]);
-            if (rows.length === 0) throw new Error(`File not found: ${id}`);
-            const meta = rows[0];
-
-            const bufSize = 16384;
-            if (options.as === 'buffer') {
-                // Stream LO into memory
-                await client.query('BEGIN');
-                const lom = new LargeObjectManager({ pg: client });
-                
-                const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
-                const totalSize = ro[0];
-                const stream = ro[1];
-                if (!stream) {
-                    throw new Error('Failed to open large object for reading');
-                }   
-                const chunks = [];
-                stream.on('data', (c) => chunks.push(c));
-                await new Promise((res, rej) => stream.on('end', res).on('error', rej));
-                await client.query('COMMIT');
-                client.release();
-                return { meta, payload: Buffer.concat(chunks) };
-            }
-
-            if (options.as === 'path') {
-                throw new Error('options.as="path" is not supported by Postgres provider');
-            }
-
-            // default: stream – wrap LO stream so we can close txn when done
-            await client.query('BEGIN');
-            const lom = new LargeObjectManager({ pg: client });
-            const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
-            const totalSize = ro[0];
-            const stream = ro[1];
-            const pass = new PassThrough();
-            stream.pipe(pass);
-            const done = new Promise((res, rej) => pass.on('end', res).on('error', rej));
-            done.finally(async () => {
-                await client.query('COMMIT').catch(() => {});
-                client.release();
-            });
-            
-            // Do not release here; we release in finally of wrapper. We return early, so prevent double release.
-            return { meta, payload: pass };
-        } catch (err) {
-            
-            throw err;
-        }
-    }
-
-    async delete(id) {
-        const client = await this.pool.connect();
-        try {
-            await client.query('BEGIN');
-            const { rows } = await client.query(`DELETE FROM ${this.schema}.${this.table} WHERE id=$1 RETURNING loid`, [
-                id,
-            ]);
-            if (rows.length) {
-                const lom = new LargeObjectManager({ pg: client });
-                await lom.unlinkAsync(rows[0].loid);
-            }
-            await client.query('COMMIT');
-        } catch (err) {
-            await client.query('ROLLBACK').catch(() => {});
-            throw err;
-        } finally {
-            client.release();
-        }
-    }
-}
-
-module.exports = PgProvider;