@flowfuse/file-server 1.14.0

package/forge/drivers/localfs.js

@@ -0,0 +1,142 @@
+const fs = require('fs')
+const { join, isAbsolute, dirname, sep: pathSeparator, parse } = require('path')
+
+const canary = 'ROOT_DIR_CANARY'
+
+async function readDirSize (dir) {
+    if (!fs.existsSync(dir)) {
+        return 0
+    }
+    const files = fs.readdirSync(dir, { withFileTypes: true })
+    const paths = files.map(async file => {
+        const path = join(dir, file.name)
+        if (file.isDirectory()) return await readDirSize(path)
+
+        if (file.isFile()) {
+            const { size } = fs.statSync(path)
+            return size
+        }
+        return 0
+    })
+
+    return (await Promise.all(paths)).flat(Infinity).reduce((i, size) => i + size, 0)
+}
+
+function cleanError (error, path, op) {
+    if (error.code === 'ENOENT') {
+        const err = new Error(`ENOENT: no such file or directory, ${op || 'stat'} '${path}'`)
+        err.code = error.code
+        return err
+    } else if (error.code === 'EPERM') {
+        const err = new Error(`EPERM: operation not permitted, ${op || 'unlink'} '${path}'`)
+        err.code = error.code
+        return err
+    }
+    return error
+}
+
+module.exports = function (app) {
+    let rootPath
+    if (!isAbsolute(app.config.driver.options.root)) {
+        rootPath = join(app.config.home, app.config.driver.options?.root)
+    } else {
+        rootPath = app.config.driver.options.root
+    }
+
+    if (!fs.existsSync(rootPath)) {
+        fs.mkdirSync(rootPath, { recursive: true })
+    }
+
+    app.log.info(`FlowForge File Storage localfs root Directory: ${rootPath}`)
+
+    function resolvePath (teamId, projectId, path) {
+        let resolvedPath
+        if (isAbsolute(path)) {
+            const pp = parse(path)
+            const hasWinDrive = /[a-z|A-Z]:/.test(pp.root)
+            if (hasWinDrive) {
+                path = path.replace(':', '')
+            }
+        }
+        const minPath = join(canary, teamId, projectId, path) // limit traversal to /teamid/projectId/**
+        if (path.startsWith(pathSeparator)) {
+            resolvedPath = join(canary, teamId, path)
+        } else {
+            resolvedPath = join(canary, teamId, projectId, path)
+        }
+        if (resolvedPath.startsWith(minPath)) {
+            const array = resolvedPath.split(pathSeparator)
+            array.shift()
+            resolvedPath = array.join(pathSeparator)
+            return resolvedPath
+        } else {
+            const err = new Error('Invalid Path')
+            err.code = 'ENOTDIR'
+            throw err
+        }
+    }
+
+    return {
+        get rootPath () {
+            return rootPath
+        },
+        resolvePath,
+        async ensureDir (teamId, projectId, path) {
+            try {
+                resolvePath(teamId, projectId, path)
+                return true
+            } catch (error) {
+                throw cleanError(error, path, 'stat')
+            }
+        },
+        async save (teamId, projectId, path, data) {
+            const fullPath = join(rootPath, resolvePath(teamId, projectId, path))
+            try {
+                fs.mkdirSync(dirname(fullPath), {
+                    recursive: true
+                })
+                fs.writeFileSync(fullPath, data)
+            } catch (error) {
+                throw cleanError(error, path, 'open')
+            }
+        },
+
+        async append (teamId, projectId, path, data) {
+            const fullPath = join(rootPath, resolvePath(teamId, projectId, path))
+            if (fs.existsSync(fullPath)) {
+                try {
+                    fs.mkdirSync(dirname(fullPath), {
+                        recursive: true
+                    })
+                    fs.appendFileSync(fullPath, data)
+                } catch (error) {
+                    throw cleanError(error, path, 'open')
+                }
+            } else {
+                await this.save(teamId, projectId, path, data)
+            }
+        },
+
+        async read (teamId, projectId, path) {
+            const fullPath = join(rootPath, resolvePath(teamId, projectId, path))
+            try {
+                return fs.readFileSync(fullPath)
+            } catch (error) {
+                throw cleanError(error, path, 'open')
+            }
+        },
+
+        async delete (teamId, projectId, path) {
+            const fullPath = join(rootPath, resolvePath(teamId, projectId, path))
+            try {
+                fs.rmSync(fullPath)
+            } catch (error) {
+                throw cleanError(error, path, 'unlink')
+            }
+        },
+
+        async quota (teamId, projectId) {
+            return readDirSize(join(rootPath, teamId, projectId))
+        }
+    }
+}

package/forge/drivers/memory.js

@@ -0,0 +1,72 @@
+const { join, sep: pathSeparator } = require('path')
+
+const canary = 'ROOT_DIR_CANARY'
+const storage = {}
+
+function resolvePath (teamId, projectId, path) {
+    let resolvedPath
+    const minPath = join(canary, teamId, projectId, path) // limit traversal to /teamid/projectId/**
+    if (path.startsWith(pathSeparator)) {
+        resolvedPath = join(canary, teamId, path)
+    } else {
+        resolvedPath = join(canary, teamId, projectId, path)
+    }
+    if (resolvedPath.startsWith(minPath)) {
+        const array = resolvedPath.split(pathSeparator)
+        array.shift()
+        resolvedPath = array.join(pathSeparator)
+        return resolvedPath
+    } else {
+        const err = new Error('Invalid Path')
+        err.code = 'ENOTDIR'
+        throw err
+    }
+}
+
+module.exports = function (app) {
+    return {
+        get rootPath () {
+            return ''
+        },
+        resolvePath,
+
+        async ensureDir (teamId, projectId, path) {
+            try {
+                resolvePath(teamId, projectId, path)
+                return true
+            } catch (error) {
+                return false
+            }
+        },
+
+        async save (teamId, projectId, path, data) {
+            storage[resolvePath(teamId, projectId, path)] = data
+        },
+
+        async append (teamId, projectId, path, data) {
+            if (storage[resolvePath(teamId, projectId, path)]) {
+                storage[resolvePath(teamId, projectId, path)] += data
+            } else {
+                storage[resolvePath(teamId, projectId, path)] = data
+            }
+        },
+
+        async read (teamId, projectId, path) {
+            return storage[resolvePath(teamId, projectId, path)]
+        },
+
+        async delete (teamId, projectId, path) {
+            delete storage[resolvePath(teamId, projectId, path)]
+        },
+
+        async quota (teamId, projectId) {
+            let used = 0
+            Object.keys(storage).forEach(key => {
+                if (key.startsWith(join(teamId, projectId))) {
+                    used += storage[key].length
+                }
+            })
+            return used
+        }
+    }
+}

package/forge/drivers/s3.js

@@ -0,0 +1,143 @@
+const canary = 'ROOT_DIR_CANARY'
+const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListObjectsCommand, HeadObjectCommand } = require('@aws-sdk/client-s3')
+const { join, isAbsolute, sep: pathSeparator, parse } = require('path')
+
+function resolvePath (teamId, projectId, path) {
+    let resolvedPath
+    if (isAbsolute(path)) {
+        const pp = parse(path)
+        const hasWinDrive = /[a-z|A-Z]:/.test(pp.root)
+        if (hasWinDrive) {
+            path = path.replace(':', '')
+        }
+    }
+    const minPath = join(canary, teamId, projectId, path) // limit traversal to /teamid/projectId/**
+    if (path.startsWith(pathSeparator)) {
+        resolvedPath = join(canary, teamId, path)
+    } else {
+        resolvedPath = join(canary, teamId, projectId, path)
+    }
+    if (resolvedPath.startsWith(minPath)) {
+        const array = resolvedPath.split(pathSeparator)
+        array.shift()
+        resolvedPath = array.join(pathSeparator)
+        return resolvedPath
+    } else {
+        const err = new Error('Invalid Path')
+        err.code = 'ENOTDIR'
+        throw err
+    }
+}
+
+module.exports = function (app) {
+    const options = app.config.driver.options
+    const bucketID = options.bucket
+    delete options.bucket
+    const s3 = new S3Client(options)
+
+    return {
+        get rootPath () {
+            return ''
+        },
+        resolvePath,
+        async ensureDir (teamId, projectId, path) {
+            const resolvedPath = resolvePath(teamId, projectId, path)
+            await s3.send(new ListObjectsCommand({
+                Bucket: bucketID,
+                Prefix: resolvedPath
+            }))
+            return true
+        },
+        async save (teamId, projectId, path, data) {
+            const resolvedPath = resolvePath(teamId, projectId, path)
+            await s3.send(new PutObjectCommand({
+                Bucket: bucketID,
+                Key: resolvedPath,
+                Body: data
+            }))
+        },
+        async append (teamId, projectId, path, data) {
+            const resolvedPath = resolvePath(teamId, projectId, path)
+
+            try {
+                await s3.send(new HeadObjectCommand({
+                    Bucket: bucketID,
+                    Key: resolvedPath
+                }))
+                // file exists so load it and append
+                try {
+                    const file = await s3.send(new GetObjectCommand({
+                        Bucket: bucketID,
+                        Key: resolvedPath
+                    }))
+                    const stream = file.Body
+                    const body = await (new Promise((resolve, reject) => {
+                        const chunks = []
+                        stream.on('data', chunk => chunks.push(chunk))
+                        stream.once('end', () => resolve(Buffer.concat(chunks)))
+                        stream.once('error', reject)
+                    }))
+                    // I HATE this as it's basically double the file size in memory
+                    const newBody = Buffer.concat([body, data])
+                    await this.save(teamId, projectId, path, newBody)
+                } catch (err) {
+                    if (err.type === 'NoSuchKey') {
+                        const error = new Error(`ENOENT: no such file or directory, open '${path}'`)
+                        error.code = 'ENOENT'
+                        throw error
+                    } else {
+                        throw err
+                    }
+                }
+            } catch (err) {
+                await this.save(teamId, projectId, path, data)
+            }
+        },
+        async read (teamId, projectId, path) {
+            const resolvedPath = resolvePath(teamId, projectId, path)
+            try {
+                const file = await s3.send(new GetObjectCommand({
+                    Bucket: bucketID,
+                    Key: resolvedPath
+                }))
+                const stream = file.Body
+                return new Promise((resolve, reject) => {
+                    const chunks = []
+                    stream.on('data', chunk => chunks.push(chunk))
+                    stream.once('end', () => resolve(Buffer.concat(chunks)))
+                    stream.once('error', reject)
+                })
+            } catch (err) {
+                if (err.Code === 'NoSuchKey') {
+                    const error = new Error(`ENOENT: no such file or directory, open '${path}'`)
+                    error.code = 'ENOENT'
+                    throw error
+                } else {
+                    throw err
+                }
+            }
+        },
+        async delete (teamId, projectId, path) {
+            const resolvedPath = resolvePath(teamId, projectId, path)
+            await s3.send(new DeleteObjectCommand({
+                Bucket: bucketID,
+                Key: resolvedPath
+            }))
+        },
+        async quota (teamId, projectId) {
+            const objects = await s3.send(new ListObjectsCommand({
+                Bucket: bucketID,
+                Prefix: join(teamId, projectId)
+            }))
+
+            let size = 0
+            if (objects.Contents) {
+                objects.Contents.forEach(file => {
+                    size += file.Size
+                })
+            }
+
+            return size
+        }
+    }
+}

package/forge/drivers/vfs.js

@@ -0,0 +1,33 @@
+module.exports = function (app, theDriver, teamId, projectId) {
+    return {
+        get rootPath () {
+            return theDriver.rootPath
+        },
+
+        resolvePath: theDriver.resolvePath,
+
+        async ensureDir (dirName) {
+            return await theDriver.ensureDir(teamId, projectId, dirName)
+        },
+
+        async save (path, data) {
+            return await theDriver.save(teamId, projectId, path, data)
+        },
+
+        async append (path, data) {
+            return await theDriver.append(teamId, projectId, path, data)
+        },
+
+        async read (path) {
+            return await theDriver.read(teamId, projectId, path)
+        },
+
+        async delete (path) {
+            return await theDriver.delete(teamId, projectId, path)
+        },
+
+        async quota () {
+            return await theDriver.quota(teamId, projectId)
+        }
+    }
+}

package/forge/fileServer.js

@@ -0,0 +1,99 @@
+const fastify = require('fastify')
+const auth = require('./auth')
+const config = require('./config')
+const driver = require('./driver')
+const routes = require('./routes')
+const helmet = require('@fastify/helmet')
+
+module.exports = async (options = {}) => {
+    const runtimeConfig = config.init(options)
+    const loggerConfig = {
+        formatters: {
+            level: (label) => {
+                return { level: label.toUpperCase() }
+            },
+            bindings: (bindings) => {
+                return { }
+            }
+        },
+        timestamp: require('pino').stdTimeFunctions.isoTime,
+        level: runtimeConfig.logging.level,
+        serializers: {
+            res (reply) {
+                return {
+                    statusCode: reply.statusCode,
+                    request: {
+                        url: reply.request?.raw?.url,
+                        method: reply.request?.method,
+                        remoteAddress: reply.request?.ip,
+                        remotePort: reply.request?.socket.remotePort
+                    }
+                }
+            }
+        }
+    }
+    if (runtimeConfig.logging.pretty !== false) {
+        loggerConfig.transport = {
+            target: 'pino-pretty',
+            options: {
+                translateTime: "UTC:yyyy-mm-dd'T'HH:MM:ss.l'Z'",
+                ignore: 'pid,hostname',
+                singleLine: true
+            }
+        }
+    }
+
+    const server = fastify({
+        bodyLimit: 10 * 1024 * 1024, // Limit set to 10MB,
+        maxParamLength: 500,
+        trustProxy: true,
+        logger: loggerConfig
+    })
+
+    if (runtimeConfig.telemetry?.backend?.prometheus?.enabled) {
+        const metricsPlugin = require('fastify-metrics')
+        await server.register(metricsPlugin, { endpoint: '/metrics' })
+    }
+
+    server.addHook('onError', async (request, reply, error) => {
+        // Useful for debugging when a route goes wrong
+        // console.log(error.stack)
+    })
+
+    try {
+        // Config
+        await server.register(config.attach, options)
+
+        // // Setup DB
+        // await server.register(db, {})
+
+        // // Setup settings
+        // await server.register(settings, {})
+
+        // Authentication Handler
+        await server.register(auth, {})
+
+        // HTTP Server setup
+        await server.register(helmet, {
+            global: true,
+            hidePoweredBy: true,
+            hsts: false,
+            frameguard: {
+                action: 'deny'
+            }
+        })
+
+        // Driver
+        await server.register(driver, {})
+
+        // Routes
+        await server.register(routes, { logLevel: server.config.logging.http })
+
+        server.ready()
+
+        return server
+    } catch (err) {
+        server.log.error(`Failed to start: ${err.toString()}`)
+        throw err
+    }
+}

package/forge/routes/context.js

@@ -0,0 +1,186 @@
+/**
+ * Persistent Context API
+ *
+ * @namespace context
+ * @memberof forge.fileserver
+ */
+
+/** @typedef {import('fastify')} Fastify */
+/** @typedef {import('fastify').FastifyReply} FastifyReply */
+/** @typedef {import('fastify').FastifyRequest} FastifyRequest */
+
+module.exports = async function (app, opts, done) {
+    const driver = require(`../context-driver/${app.config.context.type}`)
+
+    await driver.init(app)
+
+    /**
+     * Create/Update key
+     *
+     * @name /v1/context/:projectId/:scope
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.post('/:projectId/:scope', {
+        schema: {
+            body: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        key: { type: 'string' },
+                        value: {}
+                    }
+                }
+            }
+        }
+    }, async (request, reply) => {
+        const body = request.body
+        const projectId = request.params.projectId
+        const scope = request.params.scope
+        try {
+            await driver.set(projectId, scope, body, false, request.quota?.context)
+            reply.code(200).send({})
+        } catch (error) {
+            let statusCode = error.statusCode || 400
+            if (error.code === 'over_quota') {
+                statusCode = 413
+            }
+            reply.code(statusCode).send({ error: error.message, code: error.code, limit: error.limit })
+        }
+    })
+
+    /**
+     * Write full scope value from cache
+     *
+     * @name /v1/context/:projectId/cache/:scope
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.post('/:projectId/cache/:scope', {
+        schema: {
+            body: {
+                type: 'object'
+            }
+        }
+    }, async (request, reply) => {
+        const body = request.body || {}
+        const data = []
+        for (const key in body) {
+            data.push({ key, value: body[key] })
+        }
+        const projectId = request.params.projectId
+        const scope = request.params.scope
+        try {
+            await driver.set(projectId, scope, data, true, request.quota?.context)
+            reply.code(200).send({})
+        } catch (error) {
+            let statusCode = error.statusCode || 400
+            if (error.code === 'over_quota') {
+                statusCode = 413
+            }
+            reply.code(statusCode).send({ error: error.message, code: error.code, limit: error.limit })
+        }
+    })
+
+    /**
+     * Get key
+     *
+     * @name /v1/context/:projectId/:scope?
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.get('/:projectId/:scope', {
+        schema: {
+            query: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'array',
+                        items: { type: 'string' }
+                    }
+                },
+                required: ['key']
+            }
+        }
+    }, async (request, reply) => {
+        const keys = request.query.key
+        const projectId = request.params.projectId
+        try {
+            reply.send(await driver.get(projectId, request.params.scope, keys))
+        } catch (error) {
+            reply.code(400).send(error)
+        }
+    })
+
+    /**
+     * Get all context
+     * An endpoint to handle the request to get context
+     * for loading the cache
+     *
+     * @name /v1/context/:projectId/cache
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.get('/:projectId/cache', {
+
+    }, async (request, reply) => {
+        const projectId = request.params.projectId
+        const paginationOptions = app.getPaginationOptions(request, { limit: 30 })
+        try {
+            reply.send(await driver.getAll(projectId, paginationOptions))
+        } catch (error) {
+            reply.code(400).send(error)
+        }
+    })
+
+    /**
+     * Get keys
+     *
+     * @name /v1/context/:projectId/:scope/keys
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.get('/:projectId/:scope/keys', {
+
+    }, async (request, reply) => {
+        const projectId = request.params.projectId
+        try {
+            reply.send(await driver.keys(projectId, request.params.scope))
+        } catch (error) {
+            reply.code(400).send(error)
+        }
+    })
+
+    /**
+     * Delete scope
+     *
+     * @name /v1/context/:projectId/:scope
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.delete('/:projectId/:scope', {
+
+    }, async (request, reply) => {
+        // delete store[request.params.projectId][request.params.store]
+        const projectId = request.params.projectId
+        await driver.delete(projectId, request.params.scope)
+        reply.send()
+    })
+
+    /**
+     * Clean up
+     *
+     * @name /v1/context/:projectId
+     * @static
+     * @memberof forge.fileserver.context
+     */
+    app.post('/:projectId/clean', {
+
+    }, async (request, reply) => {
+        await driver.clean(request.params.projectId, request.body)
+        reply.send()
+    })
+
+    done()
+}