@flowfuse/nr-launcher 1.13.3

package/lib/logBuffer.js

@@ -0,0 +1,57 @@
+const os = require('node:os')
+const crypto = require('crypto')
+
+const instanceId = crypto.createHash('md5').update(os.hostname()).digest('hex').substring(0, 4)
+class LogBuffer {
+    constructor (size = 1000) {
+        this.size = size
+        this.buffer = new Array(this.size)
+        this.head = 0
+        this.wrapped = false
+        this.lastLogTimestamp = 0
+        this.lastLogTimestampCount = 0
+    }
+
+    add (logEntry) {
+        if (!logEntry.ts) {
+            logEntry.ts = Date.now()
+        }
+        if (logEntry.ts === this.lastLogTimestamp) {
+            this.lastLogTimestampCount++
+        } else {
+            this.lastLogTimestamp = logEntry.ts
+            this.lastLogTimestampCount = 0
+        }
+        logEntry.ts = logEntry.ts + ('' + this.lastLogTimestampCount).padStart(4, '0')
+        if (logEntry.level === 'system') {
+            console.log(logEntry.msg)
+        }
+        if (!logEntry.src) {
+            logEntry.src = instanceId
+        }
+        this.buffer[this.head++] = logEntry
+        if (this.head === this.size) {
+            this.head = 0
+            this.wrapped = true
+        }
+        return logEntry
+    }
+
+    clear () {
+        this.buffer = new Array(this.size)
+        this.head = 0
+        this.wrapped = false
+    }
+
+    toArray () {
+        if (!this.wrapped) {
+            return this.buffer.slice(0, this.head)
+        } else {
+            const result = this.buffer.slice(this.head, this.size)
+            result.push(...this.buffer.slice(0, this.head))
+            return result
+        }
+    }
+}
+
+module.exports = LogBuffer

package/lib/resources/resourcePlugin.js

@@ -0,0 +1,20 @@
+const client = require('prom-client')
+
+module.exports = (RED) => {
+    RED.plugins.registerPlugin('ff-nr-metrics', {
+        settings: {
+            '*': { exportable: true }
+        },
+        onadd: function () {
+            const collectDefaultMetrics = client.collectDefaultMetrics
+            const Registry = client.Registry
+            const register = new Registry()
+            collectDefaultMetrics({ register })
+            RED.httpAdmin.get('/ff/metrics', async function (req, res) {
+                const metrics = await register.metrics()
+                res.send(metrics)
+            })
+            RED.log.info('FlowFuse Metrics Plugin loaded')
+        }
+    })
+}

package/lib/resources/sample.js

@@ -0,0 +1,57 @@
+const got = require('got')
+let pptf
+(async function () {
+    try {
+        pptf = (await import('parse-prometheus-text-format')).default
+    } catch (err) {
+        console.log(err)
+    }
+})();
+
+
+let lastCPUTime = 0
+
+async function sampleResources (url, time) {
+    const response = {}
+    try {
+        const res  = await got.get(url, {
+            headers: {
+                pragma: 'no-cache',
+                'Cache-Control': 'max-age=0, must-revalidate, no-cache'
+            },
+            timeout: { request: 2000 },
+            retry: { limit: 0 }
+        })
+        const parsed = pptf(res.body)
+        parsed.forEach(metric => {
+            if (metric.name === 'nodejs_heap_space_size_total_bytes') {
+                metric.metrics.forEach(v => {
+                    response.hs = parseInt(v.value)
+                })
+            } else if (metric.name === 'nodejs_heap_space_size_used_bytes') {
+                metric.metrics.forEach(v => {
+                    response.hu = parseInt(v.value)
+                })
+            } else if (metric.name === 'process_resident_memory_bytes') {
+                response.ps = parseInt(metric.metrics[0].value)/(1024*1024)
+            } else if (metric.name === 'process_cpu_seconds_total') {
+                cpuTime = parseFloat(metric.metrics[0].value)
+                if (lastCPUTime != 0) {
+                    const delta = cpuTime - lastCPUTime
+                    response.cpu = (delta/time) * 100
+                }
+                lastCPUTime = cpuTime
+            } else if (metric.name === 'nodejs_eventloop_lag_mean_seconds') {
+                response.ela = parseFloat(metric.metrics[0].value)
+            } else if (metric.name === 'nodejs_eventloop_lag_p99_seconds') {
+                response.el99 = parseFloat(metric.metrics[0].value)
+            }
+        })
+    } catch (err) {
+        response.err = err.message
+    }
+
+    return response
+}
+
+module.exports = sampleResources

package/lib/resources/sampleBuffer.js

@@ -0,0 +1,85 @@
+const os = require('node:os')
+const crypto = require('crypto')
+
+const instanceId = crypto.createHash('md5').update(os.hostname()).digest('hex').substring(0, 4)
+class SampleBuffer {
+    constructor (size = 1000) {
+        this.size = size
+        this.buffer = new Array(this.size)
+        this.head = 0
+        this.wrapped = false
+        this.lastTimestamp = 0
+        this.lastTimestampCount = 0
+    }
+
+    add (sample) {
+        if (!sample.ts) {
+            sample.ts = Date.now()
+        }
+        this.buffer[this.head++] = sample
+        if (this.head == this.size) {
+            this.head = 0
+            this.wrapped = true
+        }
+        return sample
+    }
+
+    clear () {
+        this.buffer = new Array(this.size)
+        this.head = 0
+        this.wrapped = false
+    }
+
+    toArray () {
+        if (!this.wrapped) {
+            return this.buffer.slice(0, this.head)
+        } else  {
+            const result = this.buffer.slice(this.head, this.size)
+            result.push(...this.buffer.slice(0,this.head))
+            return result
+        }
+    }
+
+    lastX (x) {
+        if (this.head > x) {
+            return this.buffer.slice(this.head - x, this.head)
+        } else {
+            if (this.wrapped) {
+                const d = x - this.head
+                const result = this.buffer.slice(this.size - d, this.size)
+                result.push(...this.buffer.slice(0, this.head))
+                return result
+            } else {
+                return this.buffer.slice(0, this.head)
+            }
+        }
+    }
+
+    avgLastX (x) {
+        const samples  = this.lastX(x)
+        const result = {}
+        let skipped = 0
+        samples.forEach(sample => {
+            if (!sample.err) {
+                for (const [key, value] of Object.entries(sample)) {
+                    if (key !== 'ts' && key !== 'err') {
+                        if (result[key]) {
+                            result[key] += value
+                        } else {
+                            result[key] = value
+                        }
+                    } 
+                }
+            } else {
+                skipped++
+            }
+        })
+        for (const [key, value] of Object.entries(result)) {
+            result[key] = value/(samples.length-skipped)
+        }
+        result.count = samples.length
+        return result
+    }
+}
+
+module.exports = SampleBuffer

package/lib/runtimeSettings.js

@@ -0,0 +1,320 @@
+function getSettingsFile (settings) {
+    const projectSettings = {
+        credentialSecret: '',
+        httpAdminRoot: '',
+        dashboardUI: '',
+        disableEditor: '',
+        codeEditor: 'monaco',
+        theme: 'forge-light',
+        page_title: 'FlowFuse',
+        page_favicon: '',
+        header_title: 'FlowFuse',
+        header_url: `url: '${settings.forgeURL}/project/${settings.projectID}'`,
+        palette: {
+            allowInstall: true,
+            nodesExcludes: [],
+            denyList: [],
+            allowList: ['*'],
+            catalogues: ['https://catalogue.nodered.org/catalogue.json']
+        },
+        modules: {
+            allowInstall: true,
+            denyList: [],
+            allowList: ['*']
+        },
+        fileStore: null,
+        projectLink: null,
+        httpNodeAuth: '',
+        setupAuthMiddleware: '',
+        httpNodeMiddleware: '',
+        tcpInAllowInboundConnections: '',
+        udpInAllowInboundConnections: '',
+        tours: true,
+        libraries: ''
+    }
+    let authMiddlewareRequired = false
+    if (settings.settings) {
+        // Template/project supplied settings
+        if (settings.settings.httpNodeAuth?.user && settings.settings.httpNodeAuth?.pass) {
+            projectSettings.httpNodeAuth = `httpNodeAuth: ${JSON.stringify(settings.settings.httpNodeAuth)},`
+        } else if (settings.settings.httpNodeAuth?.type === 'flowforge-user') {
+            authMiddlewareRequired = true
+            projectSettings.setupAuthMiddleware = `const flowforgeAuthMiddleware = require('@flowfuse/nr-launcher/authMiddleware').init({
+    type: 'flowforge-user',
+    baseURL: '${settings.baseURL}',
+    forgeURL: '${settings.forgeURL}',
+    clientID: '${settings.clientID}',
+    clientSecret: '${settings.clientSecret}'
+})`
+            projectSettings.httpNodeMiddleware = 'httpNodeMiddleware: flowforgeAuthMiddleware,'
+        }
+        if (settings.credentialSecret !== undefined) {
+            projectSettings.credentialSecret = `credentialSecret: '${settings.credentialSecret}',`
+        }
+        if (settings.settings.httpAdminRoot !== undefined) {
+            projectSettings.httpAdminRoot = `httpAdminRoot: '${settings.settings.httpAdminRoot}',`
+        }
+        if (settings.settings.dashboardUI !== undefined || authMiddlewareRequired) {
+            const dashboardSettings = []
+            if (settings.settings.dashboardUI !== undefined) {
+                dashboardSettings.push(`path: '${settings.settings.dashboardUI}'`)
+            }
+            if (authMiddlewareRequired) {
+                dashboardSettings.push('middleware: flowforgeAuthMiddleware')
+            }
+            projectSettings.dashboardUI = `ui: { ${dashboardSettings.join(', ')}},`
+        }
+        if (settings.settings.disableEditor !== undefined) {
+            projectSettings.disableEditor = `disableEditor: ${settings.settings.disableEditor},`
+        }
+        if (settings.settings.ha?.replicas >= 2) {
+            projectSettings.disableEditor = 'disableEditor: true,'
+        }
+        if (settings.settings.codeEditor) {
+            projectSettings.codeEditor = settings.settings.codeEditor
+        }
+        if (typeof settings.settings.theme === 'string') {
+            if (settings.settings.theme === '' || settings.settings.theme === 'node-red') {
+                projectSettings.theme = '' // use default node-red theme
+            } else {
+                projectSettings.theme = settings.settings.theme
+            }
+        }
+        if (settings.settings.page?.title) {
+            projectSettings.page_title = settings.settings.page.title
+        }
+        projectSettings.page_favicon = 'favicon: "/theme/css/favicon.ico",' // TODO: get from theme
+        if (settings.settings.page?.favicon) {
+            projectSettings.page_favicon = `favicon: "${settings.settings.page.favicon}",`
+        }
+        if (settings.settings.header?.title) {
+            projectSettings.header_title = settings.settings.header.title
+        }
+        if (settings.settings.header?.url) {
+            projectSettings.header_url = settings.settings.header.url
+        }
+        if (settings.settings.palette?.allowInstall !== undefined) {
+            projectSettings.palette.allowInstall = settings.settings.palette.allowInstall
+        }
+        if (settings.settings.palette?.nodesExcludes !== undefined) {
+            projectSettings.palette.nodesExcludes = settings.settings.palette.nodesExcludes.split(',').map(fn => fn.trim()).filter(fn => fn.length > 0)
+        }
+        if (settings.settings.modules?.allowInstall !== undefined) {
+            projectSettings.modules.allowInstall = settings.settings.modules.allowInstall
+        }
+        if (settings.settings.palette?.allowList !== undefined) {
+            projectSettings.palette.allowList = settings.settings.palette.allowList || ['*']
+        }
+        if (settings.settings.palette?.denyList !== undefined) {
+            projectSettings.palette.denyList = settings.settings.palette.denyList || []
+        }
+        if (settings.settings.modules?.allowList !== undefined) {
+            projectSettings.modules.allowList = settings.settings.modules.allowList || ['*']
+        }
+        if (settings.settings.modules?.denyList !== undefined) {
+            projectSettings.modules.denyList = settings.settings.modules.denyList || []
+        }
+        if (settings.allowInboundTcp === true || settings.allowInboundTcp === false) {
+            projectSettings.tcpInAllowInboundConnections = `tcpInAllowInboundConnections: ${settings.allowInboundTcp},`
+        }
+        if (settings.allowInboundUdp === true || settings.allowInboundUdp === false) {
+            projectSettings.udpInAllowInboundConnections = `udpInAllowInboundConnections: ${settings.allowInboundUdp},`
+        }
+        if (settings.settings.disableTours) {
+            projectSettings.tours = false
+        }
+    }
+
+    if (settings.features?.projectComms && settings.broker) {
+        // Enable the projectLink nodes if a broker configuration is provided
+        projectSettings.projectLink = {
+            token: settings.projectToken,
+            broker: { ...settings.broker }
+        }
+        if (settings.settings.ha?.replicas >= 2) {
+            projectSettings.projectLink.useSharedSubscriptions = true
+        }
+    }
+    let contextStorage = ''
+    if (settings.fileStore?.url) {
+        // file nodes settings
+        projectSettings.fileStore = { ...settings.fileStore }
+        projectSettings.fileStore.token ||= settings.projectToken
+        if (settings.licenseType === 'ee') {
+            // context storage settings
+            contextStorage = `contextStorage: {
+                default: "memory",
+                memory: { module: 'memory' },
+                persistent: {
+                    module: require("@flowfuse/nr-launcher/context"),
+                    config: {
+                        projectID: '${settings.projectID}',
+                        url: '${settings.fileStore.url}',
+                        token: '${settings.projectToken}'
+                    }
+                }
+            },`
+        }
+    }
+    if (projectSettings.theme) {
+        const themeSettings = {
+            launcherVersion: settings.launcherVersion,
+            forgeURL: settings.forgeURL,
+            projectURL: `${settings.forgeURL}/project/${settings.projectID}`
+        }
+        projectSettings.themeSettings = `"${projectSettings.theme}": ${JSON.stringify(themeSettings)},`
+        projectSettings.theme = `theme: '${projectSettings.theme}',`
+    }
+
+    let nodesDir = ''
+    if (settings.nodesDir && settings.nodesDir.length) {
+        nodesDir = `nodesDir: ${JSON.stringify(settings.nodesDir)},`
+    }
+    if (settings.features?.['shared-library']) {
+        const sharedLibraryConfig = {
+            id: 'flowforge-team-library',
+            type: 'flowforge-team-library',
+            label: 'Team Library',
+            icon: 'font-awesome/fa-users',
+            baseURL: settings.storageURL,
+            projectID: settings.projectID,
+            libraryID: settings.teamID,
+            token: settings.projectToken
+        }
+        projectSettings.libraries = `library: { sources: [ ${JSON.stringify(sharedLibraryConfig)} ] },`
+    }
+    if (settings.licenseType === 'ee') {
+        if (settings.settings.palette?.catalogue !== undefined) {
+            projectSettings.palette.catalogues = settings.settings.palette.catalogue
+        }
+    }
+
+    const settingsTemplate = `
+${projectSettings.setupAuthMiddleware}
+module.exports = {
+    flowFile: 'flows.json',
+    flowFilePretty: true,
+    adminAuth: require('@flowfuse/nr-launcher/adminAuth')({
+        baseURL: '${settings.baseURL}',
+        forgeURL: '${settings.forgeURL}',
+        clientID: '${settings.clientID}',
+        clientSecret: '${settings.clientSecret}'
+    }),
+    ${projectSettings.credentialSecret}
+    ${projectSettings.httpAdminRoot}
+    ${projectSettings.dashboardUI}
+    ${projectSettings.disableEditor}
+    ${projectSettings.httpNodeAuth}
+    ${projectSettings.httpNodeMiddleware}
+    ${projectSettings.tcpInAllowInboundConnections}
+    ${projectSettings.udpInAllowInboundConnections}
+    httpServerOptions: {
+        "trust proxy": true
+    },
+    storageModule: require('@flowfuse/nr-launcher/storage'),
+    httpStorage: {
+        projectID: '${settings.projectID}',
+        baseURL: '${settings.storageURL}',
+        token: '${settings.projectToken}'
+    },
+    ${contextStorage}
+    logging: {
+        console: { level: 'info', metric: false, audit: false, handler: () => {
+                const levelNames = {
+                    10: "fatal",
+                    20: "error",
+                    30: "warn",
+                    40: "info",
+                    50: "debug",
+                    60: "trace",
+                    98: "audit",
+                    99: "metric"
+                }
+                return (msg) => {
+                    let message = msg.msg;
+                    try {
+                        if (typeof message === 'object' && message !== null && message.toString() === '[object Object]' && message.message) {
+                            message = message.message;
+                        }
+                    } catch(e){
+                        message = 'Exception trying to log: '+util.inspect(message);
+                    }
+                    console.log(JSON.stringify({
+                        ts: Date.now(),
+                        level: levelNames[msg.level],
+                        type: msg.type,
+                        name: msg.name,
+                        id:msg.id,
+                        msg: message
+                    }))
+                }
+            }
+
+        },
+        auditLogger: {
+            level: 'off', audit: true, handler: require('@flowfuse/nr-launcher/auditLogger'),
+            loggingURL: '${settings.auditURL}',
+            projectID: '${settings.projectID}',
+            token: '${settings.projectToken}'
+        }
+    },
+    ${nodesDir}
+    ${projectSettings.themeSettings || ''}
+    editorTheme: {
+        ${projectSettings.theme}
+        page: {
+            title: '${projectSettings.page_title}',
+            ${projectSettings.page_favicon}
+        },
+        header: {
+            title: '${projectSettings.header_title}',
+            ${projectSettings.header_url}
+        },
+        logout: {
+            redirect: '${settings.forgeURL}/project/${settings.projectID}'
+        },
+        codeEditor: {
+            lib: '${projectSettings.codeEditor}'
+        },
+        ${projectSettings.libraries}
+        tours: ${projectSettings.tours},
+        palette: {
+            catalogues: ${JSON.stringify(projectSettings.palette.catalogues)}
+        }
+    },
+    nodesExcludes: ${JSON.stringify(projectSettings.palette.nodesExcludes)},
+    externalModules: {
+        // autoInstall: true,
+        palette: {
+            allowInstall: ${projectSettings.palette.allowInstall},
+            allowUpload: false,
+            denyList: ${JSON.stringify(projectSettings.palette.denyList)},
+            allowList: ${JSON.stringify(projectSettings.palette.allowList)}
+        },
+        modules: {
+            allowInstall: ${projectSettings.modules.allowInstall},
+            denyList: ${JSON.stringify(projectSettings.modules.denyList)},
+            allowList: ${JSON.stringify(projectSettings.modules.allowList)}
+        }
+    },
+    functionExternalModules: ${projectSettings.modules.allowInstall},
+    flowforge: {
+        forgeURL: '${settings.forgeURL}',
+        teamID:  '${settings.teamID}',
+        projectID: '${settings.projectID}',
+        launcherVersion: '${settings.launcherVersion}',
+        ${projectSettings.fileStore ? 'fileStore: ' + JSON.stringify(projectSettings.fileStore) + ',' : ''}
+        ${projectSettings.projectLink ? 'projectLink: ' + JSON.stringify(projectSettings.projectLink) : ''}
+    },
+    runtimeState: {
+        enabled: true,
+        ui: true
+    }
+}
+`
+    return settingsTemplate
+}
+
+module.exports = {
+    getSettingsFile
+}

package/lib/storage/index.js

@@ -0,0 +1,92 @@
+const got = require('got')
+
+let settings
+
+module.exports = {
+    init: (nrSettings) => {
+        settings = nrSettings.httpStorage || {}
+
+        if (Object.keys(settings) === 0) {
+            const err = Promise.reject(new Error('No settings for flow storage module found'))
+            // err.catch(err => {})
+            return err
+        }
+
+        const projectID = settings.projectID
+
+        // console.log(settings)
+        // console.log(settings.baseURL + "/" + projectID + "/")
+
+        this._client = got.extend({
+            prefixUrl: settings.baseURL + '/' + projectID + '/',
+            headers: {
+                'user-agent': 'FlowFuse HTTP Storage v0.1',
+                authorization: 'Bearer ' + settings.token
+            },
+            timeout: {
+                request: 10000
+            }
+        })
+
+        return Promise.resolve()
+    },
+    getFlows: async () => {
+        return this._client.get('flows').json()
+    },
+    saveFlows: async (flow) => {
+        return this._client.post('flows', {
+            json: flow,
+            responseType: 'json'
+        })
+    },
+    getCredentials: async () => {
+        return this._client.get('credentials').json()
+    },
+    saveCredentials: async (credentials) => {
+        return this._client.post('credentials', {
+            json: credentials,
+            responseType: 'json'
+        })
+    },
+    getSettings: () => {
+        return this._client.get('settings').json()
+    },
+    saveSettings: (settings) => {
+        return this._client.post('settings', {
+            json: settings,
+            responseType: 'json'
+        })
+    },
+    getSessions: () => {
+        this._client.get('sessions').json()
+    },
+    saveSessions: (sessions) => {
+        return this._client.post('sessions', {
+            json: sessions,
+            responseType: 'json'
+        })
+    },
+    getLibraryEntry: (type, name) => {
+        return this._client.get('library/' + type, {
+            searchParams: {
+                name
+            }
+        }).then(entry => {
+            if (entry.headers['content-type'].startsWith('application/json')) {
+                return JSON.parse(entry.body)
+            } else {
+                return entry.body
+            }
+        })
+    },
+    saveLibraryEntry: (type, name, meta, body) => {
+        return this._client.post('library/' + type, {
+            json: {
+                name,
+                meta,
+                body
+            },
+            responseType: 'json'
+        })
+    }
+}