@5minds/node-red-contrib-processcube-tools 1.2.0-feature-7ac247-mg92hchl → 1.2.0-feature-608421-mg9cjskq

package/storage/providers/postgres.js

@@ -0,0 +1,160 @@
+const { Pool } = require('pg');
+const { LargeObjectManager } = require('pg-large-object');
+const { pipeline, PassThrough } = require('stream');
+const { createHash } = require('crypto');
+const { promisify } = require('util');
+const pump = promisify(pipeline);
+
+class PgProvider {
+    constructor(opts = {}) {
+        const conString = "postgres://" + opts.username + ":" + opts.password + "@" +  opts.host + ":" + opts.port + "/" + opts.database;
+        this.connectionString = conString
+        this.schema = opts.schema || 'public';
+        this.table = opts.table || 'files';
+        this.pool = new Pool({ connectionString: this.connectionString });
+        this.pool = new Pool();
+    }
+
+    async init() {
+        const client = await this.pool.connect();
+        try {
+            await client.query(`CREATE TABLE IF NOT EXISTS ${this.schema}.${this.table} (
+        id UUID PRIMARY KEY,
+        loid OID NOT NULL,
+        filename TEXT,
+        content_type TEXT,
+        size BIGINT,
+        sha256 TEXT,
+        metadata JSONB,
+        created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+      )`);
+            await client.query(
+                `CREATE INDEX IF NOT EXISTS idx_${this.table}_created_at ON ${this.schema}.${this.table}(created_at)`,
+            );
+        } finally {
+            client.release();
+        }
+    }
+
+    async store(readable, info) {
+    const { id, filename, contentType, metadata, createdAt } = info;
+    const client = await this.pool.connect();
+    try {
+        const lom = new LargeObjectManager({pg: client});
+        await client.query('BEGIN');
+        const bufSize = 16384;
+        
+        const result = await lom.createAndWritableStreamAsync(bufSize);
+        const oid =result[0];
+        const stream = result[1];
+        if (!oid || !stream) {
+            throw new Error('Failed to create large object');
+        }
+        const hash = createHash('sha256');
+        let size = 0;
+
+        // Wir berechnen Hash und Größe "on the fly", während wir in das Large Object schreiben
+        const hashAndSize = new PassThrough();
+        hashAndSize.on('data', (chunk) => {
+            hash.update(chunk);
+            size += chunk.length;
+        });
+
+        // readable -> hashAndSize -> stream
+        const pipelinePromise = pump(readable, hashAndSize, stream);
+
+        await pipelinePromise;
+
+        const sha256 = hash.digest('hex');
+
+        await client.query(
+            `INSERT INTO ${this.schema}.${this.table} (id, loid, filename, content_type, size, sha256, metadata, created_at)
+             VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8)`,
+            [id, oid, filename, contentType, size, sha256, JSON.stringify(metadata || {}), createdAt],
+        );
+
+        await client.query('COMMIT');
+        return { size, sha256, oid };
+    } catch (err) {
+        await client.query('ROLLBACK').catch(() => {});
+        throw err;
+    } finally {
+        client.release();
+    }
+}
+
+    async get(id, options = { as: 'stream' }) {
+        const client = await this.pool.connect();
+        try {
+            const { rows } = await client.query(`SELECT * FROM ${this.schema}.${this.table} WHERE id=$1`, [id]);
+            if (rows.length === 0) throw new Error(`File not found: ${id}`);
+            const meta = rows[0];
+
+            const bufSize = 16384;
+            if (options.as === 'buffer') {
+                // Stream LO into memory
+                await client.query('BEGIN');
+                const lom = new LargeObjectManager({ pg: client });
+                
+                const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
+                const totalSize = ro[0];
+                const stream = ro[1];
+                if (!stream) {
+                    throw new Error('Failed to open large object for reading');
+                }   
+                const chunks = [];
+                stream.on('data', (c) => chunks.push(c));
+                await new Promise((res, rej) => stream.on('end', res).on('error', rej));
+                await client.query('COMMIT');
+                client.release();
+                return { meta, payload: Buffer.concat(chunks) };
+            }
+
+            if (options.as === 'path') {
+                throw new Error('options.as="path" is not supported by Postgres provider');
+            }
+
+            // default: stream – wrap LO stream so we can close txn when done
+            await client.query('BEGIN');
+            const lom = new LargeObjectManager({ pg: client });
+            const ro = await lom.openAndReadableStreamAsync(meta.loid, bufSize);
+            const totalSize = ro[0];
+            const stream = ro[1];
+            const pass = new PassThrough();
+            stream.pipe(pass);
+            const done = new Promise((res, rej) => pass.on('end', res).on('error', rej));
+            done.finally(async () => {
+                await client.query('COMMIT').catch(() => {});
+                client.release();
+            });
+            
+            // Do not release here; we release in finally of wrapper. We return early, so prevent double release.
+            return { meta, payload: pass };
+        } catch (err) {
+            
+            throw err;
+        }
+    }
+
+    async delete(id) {
+        const client = await this.pool.connect();
+        try {
+            await client.query('BEGIN');
+            const { rows } = await client.query(`DELETE FROM ${this.schema}.${this.table} WHERE id=$1 RETURNING loid`, [
+                id,
+            ]);
+            if (rows.length) {
+                const lom = new LargeObjectManager({ pg: client });
+                await lom.unlinkAsync(rows[0].loid);
+            }
+            await client.query('COMMIT');
+        } catch (err) {
+            await client.query('ROLLBACK').catch(() => {});
+            throw err;
+        } finally {
+            client.release();
+        }
+    }
+}
+
+module.exports = PgProvider;

package/storage/storage-core.js

@@ -0,0 +1,77 @@
+const { v4: uuidv4 } = require('uuid');
+const { Readable } = require('stream');
+const FsProvider = require('./providers/fs');
+const PgProvider = require('./providers/postgres');
+
+function ensureReadable(payload) {
+    if (!payload && payload !== 0) {
+        throw new Error('No payload provided for storage');
+    }
+    if (Buffer.isBuffer(payload) || typeof payload === 'string' || typeof payload === 'number') {
+        return Readable.from(Buffer.isBuffer(payload) ? payload : Buffer.from(String(payload)));
+    }
+    if (payload && typeof payload.pipe === 'function') {
+        return payload; // Readable stream
+    }
+    throw new Error('Unsupported payload type. Use Buffer, string, number, or Readable stream.');
+}
+
+class StorageCore {
+    /**
+     * @param {Object} config
+     * @param {('fs'|'pg')} config.provider
+     * @param {Object} [config.fs]
+     * @param {string} [config.fs.baseDir]
+     * @param {Object} [config.pg]
+     * @param {string} [config.pg.username]
+     * @param {string} [config.pg.password]
+     * @param {string} [config.pg.host]
+     * @param {number} [config.pg.port]
+     * @param {string} [config.pg.database]
+     * @param {string} [config.pg.schema]
+     * @param {string} [config.pg.table]
+     */
+    constructor(config = {}) {
+        this.config = config;
+        const p = config.provider || 'fs';
+        if (p === 'fs') this.provider = new FsProvider(config.fs || {});
+        else if (p === 'pg') this.provider = new PgProvider(config.pg || {});
+        else throw new Error(`Unknown provider: ${p}`);
+    }
+
+    async init() {
+        if (typeof this.provider.init === 'function') {
+            await this.provider.init();
+        }
+    }
+
+    /** Store a file */
+    async store(payload, file = {}) {
+        const stream = ensureReadable(payload);
+        const id = uuidv4();
+        const info = {
+            id,
+            filename: file.filename || id,
+            contentType: file.contentType || 'application/octet-stream',
+            metadata: file.metadata || {},
+            createdAt: new Date().toISOString(),
+        };
+        const result = await this.provider.store(stream, info);
+        return { ...info, ...result, storage: this.config.provider || 'fs' };
+    }
+
+    /** Get a file by id */
+    async get(id, options = { as: 'stream' }) {
+        if (!id) throw new Error('id is required');
+        return this.provider.get(id, options);
+    }
+
+    /** Delete a file by id */
+    async delete(id) {
+        if (!id) throw new Error('id is required');
+        await this.provider.delete(id);
+        return { id, deleted: true };
+    }
+}
+
+module.exports = StorageCore;

package/test/helpers/email-receiver.mocks.js

@@ -0,0 +1,447 @@
+/**
+ * Shared mock objects and utilities for Email Receiver Node tests
+ */
+
+/**
+ * Create mock Node-RED object for unit testing
+ */
+function createMockNodeRED(options = {}) {
+    // Store input callback in the mock RED context
+    let storedInputCallback = null;
+    let nodeInstance = null;
+
+    const mockRED = {
+        nodes: {
+            createNode: function (node, config) {
+                nodeInstance = node; // Capture the node instance
+
+                // Apply config properties to node
+                Object.assign(node, {
+                    id: config.id || 'mock-node-id',
+                    type: config.type || 'email-receiver',
+                    name: config.name || 'Mock Node',
+                    on: function (event, callback) {
+                        if (event === 'input') {
+                            storedInputCallback = callback;
+                            // Store the callback on the node instance for easy access
+                            node.inputCallback = callback;
+                        }
+                        // Call the original onHandler if provided
+                        if (options.onHandler) {
+                            options.onHandler.call(node, event, callback);
+                        }
+                    },
+                    status: options.statusHandler || function () {},
+                    error: options.errorHandler || function () {},
+                    send: options.sendHandler || function () {},
+                    log: options.logHandler || function () {},
+                    warn: options.warnHandler || function () {},
+                    debug: options.debugHandler || function () {},
+                });
+                return node;
+            },
+            registerType: function (type, constructor) {
+                // Store registration for verification in tests
+                this.lastRegisteredType = type;
+                this.lastRegisteredConstructor = constructor;
+            },
+            // Helper method to get the stored input callback
+            getInputCallback: function () {
+                return storedInputCallback;
+            },
+            // Helper method to get the node instance
+            getNodeInstance: function () {
+                return nodeInstance;
+            },
+        },
+        util: {
+            evaluateNodeProperty: function (value, type, node, msg, callback) {
+                if (type === 'json') {
+                    try {
+                        // Simulate parsing a JSON string into an object
+                        return JSON.parse(JSON.stringify(value));
+                    } catch (e) {
+                        if (callback) {
+                            callback(e, null);
+                        }
+                        return null;
+                    }
+                }
+
+                // Simple mock implementation
+                if (callback) {
+                    callback(null, value);
+                }
+                return value;
+            },
+            encrypt: function (value) {
+                return 'encrypted:' + value;
+            },
+            decrypt: function (value) {
+                return value.replace('encrypted:', '');
+            },
+        },
+        log: {
+            info: options.logInfo || function () {},
+            warn: options.logWarn || function () {},
+            error: options.logError || function () {},
+            debug: options.logDebug || function () {},
+        },
+    };
+
+    return mockRED;
+}
+
+/**
+ * Mock IMAP implementation for testing
+ */
+function createMockImap() {
+    return function MockImap(config) {
+        this.config = config;
+        this.events = {};
+
+        // Simulate connection behavior
+        this.connect = () => {
+            // Check if we should simulate a connection error
+            if (this.config.host && this.config.host.includes('invalid')) {
+                // Simulate connection error
+                if (this.events && this.events.error) {
+                    setTimeout(() => {
+                        const error = new Error('Connection failed');
+                        error.code = 'ENOTFOUND';
+                        this.events.error(error);
+                    }, 10);
+                }
+            } else {
+                // Simulate successful connection by emitting 'ready' event
+                if (this.events && this.events.ready) {
+                    setTimeout(() => this.events.ready(), 10);
+                }
+            }
+        };
+
+        // Simulate opening a mailbox
+        this.openBox = (folder, readOnly, callback) => {
+            setTimeout(() => {
+                callback(null, {
+                    messages: { total: 1 },
+                    name: folder,
+                    readOnly: readOnly,
+                });
+            }, 10);
+        };
+
+        // Simulate searching for emails
+        this.search = (criteria, callback) => {
+            setTimeout(() => {
+                // Return mock message IDs
+                callback(null, [123, 456, 789]);
+            }, 10);
+        };
+
+        // Simulate fetching email messages
+        this.fetch = (results, options) => {
+            const fetchEmitter = {
+                on: (event, callback) => {
+                    if (event === 'message') {
+                        setTimeout(() => {
+                            const mockMessage = {
+                                on: (messageEvent, messageCallback) => {
+                                    if (messageEvent === 'body') {
+                                        setTimeout(() => {
+                                            // Return a mock email body that will be parsed
+                                            const mockEmailContent = `From: sender@test.com\r\nTo: recipient@test.com\r\nSubject: Mock Email Subject\r\n\r\nThis is a mock email body for testing purposes.`;
+                                            messageCallback(Buffer.from(mockEmailContent));
+                                        }, 5);
+                                    } else if (messageEvent === 'attributes') {
+                                        setTimeout(() => {
+                                            messageCallback({
+                                                uid: 123,
+                                                flags: ['\\Seen'],
+                                                date: new Date(),
+                                                size: 1024,
+                                            });
+                                        }, 5);
+                                    }
+                                },
+                                once: (messageEvent, messageCallback) => {
+                                    if (messageEvent === 'end') {
+                                        setTimeout(() => messageCallback(), 15);
+                                    }
+                                },
+                            };
+                            callback(mockMessage);
+                        }, 10);
+                    }
+                },
+                once: (event, callback) => {
+                    if (event === 'end') {
+                        setTimeout(() => callback(), 20);
+                    } else if (event === 'error') {
+                        // Store error callback for potential use
+                        this.errorCallback = callback;
+                    }
+                },
+            };
+
+            // Store the fetch emitter so we can trigger events manually in tests
+            this.lastFetchEmitter = fetchEmitter;
+            return fetchEmitter;
+        };
+
+        // Simulate closing connection
+        this.end = () => {
+            if (this.events && this.events.end) {
+                setTimeout(() => this.events.end(), 5);
+            }
+        };
+
+        // Event listener setup
+        this.once = (event, callback) => {
+            if (!this.events) this.events = {};
+            this.events[event] = callback;
+        };
+
+        this.on = (event, callback) => {
+            if (!this.events) this.events = {};
+            this.events[event] = callback;
+        };
+
+        // Additional IMAP methods that might be used
+        this.addFlags = (source, flags, callback) => {
+            setTimeout(() => callback(null), 5);
+        };
+
+        this.removeFlags = (source, flags, callback) => {
+            setTimeout(() => callback(null), 5);
+        };
+
+        // Helper method to trigger the email processing flow
+        this.simulateNewEmail = (emailData = {}) => {
+            if (this.events && this.events.mail) {
+                setTimeout(() => {
+                    this.events.mail(1); // Simulate 1 new email
+                }, 10);
+            }
+        };
+
+        return this;
+    };
+}
+
+/**
+ * Mock Mailparser implementation for testing
+ */
+function createMockMailparser() {
+    return {
+        simpleParser: function (source, options = {}) {
+            return Promise.resolve({
+                subject: options.subject || 'Mock Email Subject',
+                text: options.text || 'This is a mock email body for testing purposes.',
+                html: options.html || '<p>This is a mock email body for testing purposes.</p>',
+                from: {
+                    text: options.from || 'sender@test.com',
+                    value: [{ address: options.from || 'sender@test.com', name: 'Test Sender' }],
+                },
+                to: {
+                    text: options.to || 'recipient@test.com',
+                    value: [{ address: options.to || 'recipient@test.com', name: 'Test Recipient' }],
+                },
+                date: options.date || new Date(),
+                messageId: options.messageId || '<mock-message-id@test.com>',
+                headers: new Map([
+                    ['message-id', '<mock-message-id@test.com>'],
+                    ['subject', options.subject || 'Mock Email Subject'],
+                    ['from', options.from || 'sender@test.com'],
+                    ['to', options.to || 'recipient@test.com'],
+                ]),
+                attachments: options.attachments || [],
+            });
+        },
+    };
+}
+
+/**
+ * Enhanced module mocks setup with better email simulation
+ */
+function setupModuleMocks() {
+    const mockModules = {
+        'node-imap': createMockImap(),
+        mailparser: createMockMailparser(),
+    };
+
+    const Module = require('module');
+    const originalLoad = Module._load;
+
+    Module._load = function (request, parent) {
+        if (mockModules[request]) {
+            return mockModules[request];
+        }
+        return originalLoad.apply(this, arguments);
+    };
+
+    // Return cleanup function
+    return function cleanup() {
+        Module._load = originalLoad;
+    };
+}
+
+/**
+ * Create test configurations for different scenarios
+ */
+const testConfigs = {
+    valid: {
+        id: 'test-node-1',
+        type: 'email-receiver',
+        name: 'Test Email Receiver',
+        host: 'imap.test.com',
+        hostType: 'str',
+        port: 993,
+        portType: 'num',
+        tls: true,
+        tlsType: 'bool',
+        user: 'test@test.com',
+        userType: 'str',
+        password: 'testpass',
+        passwordType: 'str',
+        folder: ['INBOX'],
+        folderType: 'str',
+        markseen: true,
+        markseenType: 'bool',
+    },
+
+    arrayFolders: {
+        id: 'test-node-3',
+        type: 'email-receiver',
+        name: 'Array Folders Test',
+        host: 'imap.test.com',
+        hostType: 'str',
+        port: 993,
+        portType: 'num',
+        user: 'test@test.com',
+        userType: 'str',
+        password: 'testpass',
+        passwordType: 'str',
+        folder: ['INBOX', 'Junk', 'Drafts'],
+        folderType: 'json',
+        markseen: false,
+        markseenType: 'bool',
+    },
+
+    invalidFolderType: {
+        id: 'test-node-4',
+        type: 'email-receiver',
+        name: 'Invalid Config Test',
+        host: '', // Missing host
+        hostType: 'str',
+        port: 993,
+        portType: 'num',
+        user: 'test@test.com',
+        userType: 'str',
+        password: '', // Missing password
+        passwordType: 'str',
+        folder: 123,
+        folderType: 'num',
+    },
+
+    invalidConfig: {
+        id: 'test-node-4',
+        type: 'email-receiver',
+        name: 'Invalid Config Test',
+        host: '', // Missing host
+        hostType: 'str',
+        port: 993,
+        portType: 'num',
+        user: 'test@test.com',
+        userType: 'str',
+        password: '', // Missing password
+        passwordType: 'str',
+        folder: ['Inbox'],
+        folderType: 'num',
+    },
+
+    minimal: {
+        id: 'test-node-5',
+        type: 'email-receiver',
+        host: 'imap.minimal.com',
+        hostType: 'str',
+        port: 993,
+        portType: 'num',
+        user: 'minimal@test.com',
+        userType: 'str',
+        password: 'minimalpass',
+        passwordType: 'str',
+        folder: 'INBOX',
+        folderType: 'str',
+    },
+};
+
+/**
+ * Create test flows for Node-RED integration tests
+ */
+const testFlows = {
+    single: [testConfigs.valid],
+
+    withHelper: [testConfigs.valid, { id: 'h1', type: 'helper' }],
+
+    connected: [
+        { ...testConfigs.valid, wires: [['h1']] },
+        { id: 'h1', type: 'helper' },
+    ],
+
+    multiOutput: [
+        { ...testConfigs.valid, wires: [['h1', 'h2']] },
+        { id: 'h1', type: 'helper' },
+        { id: 'h2', type: 'helper' },
+    ],
+};
+
+/**
+ * Utility functions for test assertions and email simulation
+ */
+const testUtils = {
+    /**
+     * Wait for a specified amount of time
+     */
+    wait: (ms = 100) => new Promise((resolve) => setTimeout(resolve, ms)),
+
+    /**
+     * Create a promise that resolves when a node receives a message
+     */
+    waitForMessage: (node, timeout = 1000) => {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                reject(new Error('Timeout waiting for message'));
+            }, timeout);
+
+            node.on('input', (msg) => {
+                clearTimeout(timer);
+                resolve(msg);
+            });
+        });
+    },
+
+    /**
+     * Verify that a message has expected properties
+     */
+    verifyMessage: (msg, expectedProps = {}) => {
+        const should = require('should');
+        should.exist(msg);
+
+        Object.keys(expectedProps).forEach((prop) => {
+            if (expectedProps[prop] !== undefined) {
+                msg.should.have.property(prop, expectedProps[prop]);
+            }
+        });
+    },
+};
+
+module.exports = {
+    createMockNodeRED,
+    createMockImap,
+    createMockMailparser,
+    setupModuleMocks,
+    testConfigs,
+    testFlows,
+    testUtils,
+};