@flowfuse/nr-file-nodes 0.0.3

package/file.js

@@ -0,0 +1,402 @@
+/**
+ * Copyright JS Foundation and other contributors, http://js.foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+module.exports = function (RED) {
+    'use strict'
+
+    // Do not register nodes in runtime if settings are not provided
+    if (
+        !RED.settings.flowforge ||
+        !RED.settings.flowforge.projectID ||
+        !RED.settings.flowforge.teamID ||
+        !RED.settings.flowforge.fileStore ||
+        !RED.settings.flowforge.fileStore.url
+    ) {
+        throw new Error('FlowFuse file nodes cannot be loaded without required settings')
+    }
+    const VFS = require('./vfs')
+    const os = require('os')
+    const path = require('path')
+    const iconv = require('iconv-lite')
+
+    function encode (data, enc) {
+        if (enc !== 'none') {
+            return iconv.encode(data, enc)
+        }
+        return Buffer.from(data)
+    }
+
+    function decode (data, enc) {
+        if (enc !== 'none') {
+            return iconv.decode(data, enc)
+        }
+        return data.toString()
+    }
+
+    function FileNode (n) {
+        // Write/delete a file
+        RED.nodes.createNode(this, n)
+        this.filename = n.filename
+        this.filenameType = n.filenameType
+        this.appendNewline = n.appendNewline
+        this.overwriteFile = n.overwriteFile.toString()
+        this.createDir = n.createDir || false
+        this.encoding = n.encoding || 'none'
+        const node = this
+        const fs = VFS(RED)
+        node.wstream = null
+        node.msgQueue = []
+        node.closing = false
+        node.closeCallback = null
+
+        function processMsg (msg, nodeSend, done) {
+            let filename = node.filename || ''
+            // Pre V3 compatibility - if filenameType is empty, do in place upgrade
+            if (typeof node.filenameType === 'undefined' || node.filenameType === '') {
+                // existing node AND filenameType is not set - inplace (compatible) upgrade
+                if (filename === '') { // was using empty value to denote msg.filename
+                    node.filename = 'filename'
+                    node.filenameType = 'msg'
+                } else { // was using a static filename - set typedInput type to str
+                    node.filenameType = 'str'
+                }
+            }
+
+            RED.util.evaluateNodeProperty(node.filename, node.filenameType, node, msg, (err, value) => {
+                if (err) {
+                    node.error(err, msg)
+                    return done()
+                } else {
+                    filename = value
+                }
+            })
+            filename = filename || ''
+            msg.filename = filename
+            let fullFilename = filename
+            if (filename && RED.settings.fileWorkingDirectory && !path.isAbsolute(filename)) {
+                // fullFilename = path.resolve(path.join(RED.settings.fileWorkingDirectory, filename))
+                fullFilename = path.join(RED.settings.fileWorkingDirectory, filename)
+            }
+            if ((!node.filename) && (!node.tout)) {
+                node.tout = setTimeout(function () {
+                    node.status({ fill: 'grey', shape: 'dot', text: filename })
+                    clearTimeout(node.tout)
+                    node.tout = null
+                }, 333)
+            }
+            if (path.isAbsolute(fullFilename)) {
+                fullFilename = fullFilename.slice(1)
+            }
+            if (filename === '') {
+                node.warn(RED._('file.errors.nofilename'))
+                done()
+            } else if (node.overwriteFile === 'delete') {
+                fs.unlink(fullFilename, function (err) {
+                    if (err) {
+                        node.error(RED._('file.errors.deletefail', { error: err.toString() }), msg)
+                    } else {
+                        node.debug(RED._('file.status.deletedfile', { file: filename }))
+                        nodeSend(msg)
+                    }
+                    done()
+                })
+            // eslint-disable-next-line no-prototype-builtins
+            } else if (msg.hasOwnProperty('payload') && (typeof msg.payload !== 'undefined')) {
+                async function ensureDir (name, successCallback) {
+                    const dir = path.dirname(name)
+                    if (node.createDir) {
+                        fs.ensureDir(dir, function (err) {
+                            if (err) {
+                                node.error(RED._('file.errors.createfail', { error: err.toString() }), msg)
+                            }
+                            successCallback()
+                        })
+                    } else {
+                        successCallback()
+                    }
+                }
+
+                ensureDir(fullFilename, function success () {
+                    let data = msg.payload
+                    if ((typeof data === 'object') && (!Buffer.isBuffer(data))) {
+                        data = JSON.stringify(data)
+                    }
+                    if (typeof data === 'boolean') { data = data.toString() }
+                    if (typeof data === 'number') { data = data.toString() }
+                    if ((node.appendNewline) && (!Buffer.isBuffer(data))) { data += os.EOL }
+                    let buf
+                    if (node.encoding === 'setbymsg') {
+                        buf = encode(data, msg.encoding || 'none')
+                    } else { buf = encode(data, node.encoding) }
+                    if (node.overwriteFile === 'true') {
+                        fs.writeFile(fullFilename, buf, function (err) {
+                            if (err) {
+                                node.error(RED._('file.errors.writefail', { error: err.toString() }), msg)
+                            } else {
+                                nodeSend(msg)
+                            }
+                            done()
+                        })
+                    } else {
+                        fs.appendFile(fullFilename, buf, function (err) {
+                            if (err) {
+                                node.error(RED._('file.errors.appendfail', { error: err.toString() }), msg)
+                            } else {
+                                nodeSend(msg)
+                            }
+                            done()
+                        })
+                    }
+                })
+            } else {
+                done()
+            }
+        }
+
+        function processQueue (queue) {
+            const event = queue[0]
+            processMsg(event.msg, event.send, function () {
+                event.done()
+                queue.shift()
+                if (queue.length > 0) {
+                    processQueue(queue)
+                } else if (node.closing) {
+                    closeNode()
+                }
+            })
+        }
+
+        this.on('input', function (msg, nodeSend, nodeDone) {
+            const msgQueue = node.msgQueue
+            msgQueue.push({
+                msg,
+                send: nodeSend,
+                done: nodeDone
+            })
+            if (msgQueue.length > 1) {
+                // pending write exists
+                return
+            }
+            try {
+                processQueue(msgQueue)
+            } catch (e) {
+                node.msgQueue = []
+                if (node.closing) {
+                    closeNode()
+                }
+                throw e
+            }
+        })
+
+        function closeNode () {
+            if (node.wstream) { node.wstream.end() }
+            if (node.tout) { clearTimeout(node.tout) }
+            node.status({})
+            const cb = node.closeCallback
+            node.closeCallback = null
+            node.closing = false
+            if (cb) {
+                cb()
+            }
+        }
+
+        this.on('close', function (done) {
+            if (node.closing) {
+                // already closing
+                return
+            }
+            node.closing = true
+            if (done) {
+                node.closeCallback = done
+            }
+            if (node.msgQueue.length > 0) {
+                // close after queue processed
+
+            } else {
+                closeNode()
+            }
+        })
+    }
+    RED.nodes.registerType('file', FileNode)
+
+    function FileInNode (n) {
+        // Read a file
+        RED.nodes.createNode(this, n)
+        this.filename = n.filename
+        this.filenameType = n.filenameType
+        this.format = n.format
+        this.chunk = false
+        this.encoding = n.encoding || 'none'
+        this.allProps = n.allProps || false
+        if (n.sendError === undefined) {
+            this.sendError = true
+        } else {
+            this.sendError = n.sendError
+        }
+        if (this.format === 'lines') { this.chunk = true }
+        if (this.format === 'stream') { this.chunk = true }
+        const node = this
+        const fs = VFS(RED)
+        this.on('input', function (msg, nodeSend, nodeDone) {
+            let filename = node.filename || ''
+            // Pre V3 compatibility - if filenameType is empty, do in place upgrade
+            if (typeof node.filenameType === 'undefined' || node.filenameType === '') {
+                // existing node AND filenameType is not set - inplace (compatible) upgrade
+                if (filename === '') { // was using empty value to denote msg.filename
+                    node.filename = 'filename'
+                    node.filenameType = 'msg'
+                } else { // was using a static filename - set typedInput type to str
+                    node.filenameType = 'str'
+                }
+            }
+            let propertyError = false
+            RED.util.evaluateNodeProperty(node.filename, node.filenameType, node, msg, (err, value) => {
+                if (err) {
+                    node.error(err, msg)
+                    propertyError = true
+                    // return done()
+                } else {
+                    filename = (value || '').replace(/\t|\r|\n/g, '')
+                }
+            })
+            if (propertyError) {
+                return
+            }
+            filename = filename || ''
+            let fullFilename = filename
+
+            if (filename && RED.settings.fileWorkingDirectory && !path.isAbsolute(filename)) {
+                // fullFilename = path.resolve(path.join(RED.settings.fileWorkingDirectory, filename))
+                fullFilename = path.join(RED.settings.fileWorkingDirectory, filename)
+            }
+            if (!node.filename) {
+                node.status({ fill: 'grey', shape: 'dot', text: filename })
+            }
+            if (path.isAbsolute(fullFilename)) {
+                fullFilename = fullFilename.slice(1)
+            }
+            if (filename === '') {
+                node.warn(RED._('file.errors.nofilename'))
+                nodeDone()
+            } else {
+                msg.filename = filename
+                let lines = Buffer.from([])
+                let spare = ''
+                let count = 0
+                let type = 'buffer'
+                let ch = ''
+                if (node.format === 'lines') {
+                    ch = '\n'
+                    type = 'string'
+                }
+                let getout = false
+
+                const rs = fs.createReadStream(fullFilename)
+                    .on('readable', function () {
+                        let chunk
+                        let m
+                        const hwm = rs._readableState.highWaterMark
+                        while ((chunk = rs.read()) !== null) {
+                            if (node.chunk === true) {
+                                getout = true
+                                if (node.format === 'lines') {
+                                    spare += decode(chunk, node.encoding)
+                                    const bits = spare.split('\n')
+                                    let i = 0
+                                    for (i = 0; i < bits.length - 1; i++) {
+                                        m = {}
+                                        if (node.allProps === true) {
+                                            m = RED.util.cloneMessage(msg)
+                                        } else {
+                                            m.topic = msg.topic
+                                            m.filename = msg.filename
+                                        }
+                                        m.payload = bits[i]
+                                        m.parts = { index: count, ch, type, id: msg._msgid }
+                                        count += 1
+                                        nodeSend(m)
+                                    }
+                                    spare = bits[i]
+                                }
+                                if (node.format === 'stream') {
+                                    m = {}
+                                    if (node.allProps === true) {
+                                        m = RED.util.cloneMessage(msg)
+                                    } else {
+                                        m.topic = msg.topic
+                                        m.filename = msg.filename
+                                    }
+                                    m.payload = chunk
+                                    m.parts = { index: count, ch, type, id: msg._msgid }
+                                    count += 1
+                                    if (chunk.length < hwm) { // last chunk is smaller that high water mark = eof
+                                        getout = false
+                                        m.parts.count = count
+                                    }
+                                    nodeSend(m)
+                                }
+                            } else {
+                                lines = Buffer.concat([lines, chunk])
+                            }
+                        }
+                    })
+                    .on('error', function (err) {
+                        node.error(err, msg)
+                        if (node.sendError) {
+                            const sendMessage = RED.util.cloneMessage(msg)
+                            delete sendMessage.payload
+                            sendMessage.error = err
+                            nodeSend(sendMessage)
+                        }
+                        nodeDone()
+                    })
+                    .on('end', function () {
+                        if (node.chunk === false) {
+                            if (node.format === 'utf8') {
+                                msg.payload = decode(lines, node.encoding)
+                            } else { msg.payload = lines }
+                            nodeSend(msg)
+                        } else if (node.format === 'lines') {
+                            let m = {}
+                            if (node.allProps) {
+                                m = RED.util.cloneMessage(msg)
+                            } else {
+                                m.topic = msg.topic
+                                m.filename = msg.filename
+                            }
+                            m.payload = spare
+                            m.parts = {
+                                index: count,
+                                count: count + 1,
+                                ch,
+                                type,
+                                id: msg._msgid
+                            }
+                            nodeSend(m)
+                        } else if (getout) { // last chunk same size as high water mark - have to send empty extra packet.
+                            const m = { parts: { index: count, count, ch, type, id: msg._msgid } }
+                            nodeSend(m)
+                        }
+                        nodeDone()
+                    })
+            }
+        })
+        this.on('close', function () {
+            node.status({})
+        })
+    }
+    RED.nodes.registerType('file in', FileInNode)
+}

package/locales/en-US/file.html

@@ -0,0 +1,69 @@
+<!--
+  Copyright JS Foundation and other contributors, http://js.foundation
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<script type="text/html" data-help-name="file">
+    <p>Writes <code>msg.payload</code> to a file, either adding to the end or replacing the existing content.
+       Alternatively, it can delete the file.</p>
+    <h3>Inputs</h3>
+    <dl class="message-properties">
+        <dt class="optional">filename <span class="property-type">string</span></dt>
+        <dd>The name of the file to be updated can be provided in the node configuration, or as a message property. 
+            By default it will use <code>msg.filename</code> but this can be customised in the node.
+        </dd>
+        <dt class="optional">encoding <span class="property-type">string</span></dt>
+        <dd>If encoding is configured to be set by msg, then this optional property can set the encoding.</dt>
+    </dl>
+    <h3>Output</h3>
+    <p>On completion of write, input message is sent to output port.</p>
+    <h3>Details</h3>
+    <p>Each message payload will be added to the end of the file, optionally appending
+    a newline (\n) character between each one.</p>
+    <p>If <code>msg.filename</code> is used the file will be closed after every write.
+    For best performance use a fixed filename.</p>
+    <p>It can be configured to overwrite the entire file rather than append. For example,
+    when writing binary data to a file, such as an image, this option should be used
+    and the option to append a newline should be disabled.</p>
+    <p>Encoding of data written to a file can be specified from list of encodings.</p>
+    <p>Alternatively, this node can be configured to delete the file.</p>
+</script>
+
+<script type="text/html" data-help-name="file in">
+    <p>Reads the contents of a file as either a string or binary buffer.</p>
+    <h3>Inputs</h3>
+    <dl class="message-properties">
+        <dt class="optional">filename <span class="property-type">string</span></dt>
+        <dd>The name of the file to be read can be provided in the node configuration, or as a message property. 
+            By default it will use <code>msg.filename</code> but this can be customised in the node.
+        </dd>
+    </dl>
+    <h3>Outputs</h3>
+    <dl class="message-properties">
+        <dt>payload <span class="property-type">string | buffer</span></dt>
+        <dd>The contents of the file as either a string or binary buffer.</dd>
+        <dt class="optional">filename <span class="property-type">string</span></dt>
+        <dd>If not configured in the node, this optional property sets the name of the file to be read.</dd>
+    </dl>
+    <h3>Details</h3>
+    <p>The filename should be an absolute path, otherwise it will be relative to
+    the working directory of the Node-RED process.</p>
+    <p>On Windows, path separators may need to be escaped, for example: <code>\\Users\\myUser</code>.</p>
+    <p>Optionally, a text file can be split into lines, outputting one message per line, or a binary file
+    split into smaller buffer chunks - the chunk size being operating system dependant, but typically 64k (Linux/Mac) or 41k (Windows).</p>
+    <p>When split into multiple messages, each message will have a <code>parts</code>
+    property set, forming a complete message sequence.</p>
+    <p>Encoding of input data can be specified from list of encodings if output format is string.</p>
+    <p>Errors should be caught and handled using a Catch node.</p>
+</script>

package/locales/en-US/file.json

@@ -0,0 +1,65 @@
+{
+    "file": {
+        "label": {
+            "write": "write file",
+            "read": "read file",
+            "filename": "Filename",
+            "path": "path",
+            "action": "Action",
+            "addnewline": "Add newline (\\n) to each payload?",
+            "createdir": "Create directory if it doesn't exist?",
+            "outputas": "Output",
+            "breakchunks": "Break into chunks",
+            "breaklines": "Break into lines",
+            "sendError": "Send message on error (legacy mode)",
+            "encoding": "Encoding",
+            "deletelabel": "delete __file__",
+            "utf8String": "UTF8 string",
+            "utf8String_plural": "UTF8 strings",
+            "binaryBuffer": "binary buffer",
+            "binaryBuffer_plural": "binary buffers",
+            "allProps": "include all existing properties in each msg"
+        },
+        "action": {
+            "append": "append to file",
+            "overwrite": "overwrite file",
+            "delete": "delete file"
+        },
+        "output": {
+            "utf8": "a single utf8 string",
+            "buffer": "a single Buffer object",
+            "lines": "a msg per line",
+            "stream": "a stream of Buffers"
+        },
+        "status": {
+            "wrotefile": "wrote to file: __file__",
+            "deletedfile": "deleted file: __file__",
+            "appendedfile": "appended to file: __file__"
+        },
+        "encoding": {
+            "none": "default",
+            "setbymsg": "set by msg.encoding",
+            "native": "Native",
+            "unicode": "Unicode",
+            "japanese": "Japanese",
+            "chinese": "Chinese",
+            "korean": "Korean",
+            "taiwan": "Taiwan/Hong Kong",
+            "windows": "Windows codepages",
+            "iso": "ISO codepages",
+            "ibm": "IBM codepages",
+            "mac": "Mac codepages",
+            "koi8": "KOI8 codepages",
+            "misc": "Miscellaneous"
+        },
+        "errors": {
+            "nofilename": "No filename specified",
+            "invaliddelete": "Warning: Invalid delete. Please use specific delete option in config dialog.",
+            "deletefail": "failed to delete file: __error__",
+            "writefail": "failed to write to file: __error__",
+            "appendfail": "failed to append to file: __error__",
+            "createfail": "failed to create file: __error__"
+        },
+        "tip": "Tip: The filename should be an absolute path, otherwise it will be relative to the working directory of the Node-RED process."
+    }
+}

package/package.json

@@ -0,0 +1,54 @@
+{
+    "name": "@flowfuse/nr-file-nodes",
+    "version": "0.0.3",
+    "description": "Node-RED file nodes packaged for FlowFuse",
+    "main": "index.js",
+    "scripts": {
+        "test": "npm run test:files && npm run test:memory",
+        "test:memory": "mocha 'test/memory_spec.js' --timeout 5000",
+        "test:files": "mocha 'test/file_spec.js' --timeout 5000",
+        "lint": "eslint -c .eslintrc *.js",
+        "lint:fix": "eslint -c .eslintrc *.js --fix"
+    },
+    "keywords": [
+        "FlowFuse",
+        "node-red",
+        "filesystem"
+    ],
+    "node-red": {
+        "version": ">=3.0.0",
+        "nodes": {
+            "file": "file.js"
+        }
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/FlowFuse/nr-file-nodes.git"
+    },
+    "author": {
+        "name": "FlowFuse Inc."
+    },
+    "bugs": {
+        "url": "https://github.com/FlowFuse/nr-file-nodes/issues"
+    },
+    "homepage": "https://github.com/FlowFuse/nr-file-nodes#readme",
+    "dependencies": {
+        "got": "11.8.5",
+        "iconv-lite": "0.6.3"
+    },
+    "engines": {
+        "node": ">=16.x"
+    },
+    "devDependencies": {
+        "@flowforge/file-server": "^0.0.5",
+        "eslint": "^8.25.0",
+        "eslint-config-standard": "^17.0.0",
+        "eslint-plugin-no-only-tests": "^3.1.0",
+        "fs-extra": "^10.1.0",
+        "mocha": "^10.1.0",
+        "mocha-cli": "^1.0.1",
+        "node-red": "^3.1.0",
+        "node-red-node-test-helper": "^0.3.0",
+        "sinon": "^14.0.2"
+    }
+}