hfs 0.26.6 → 0.26.8

package/src/api.monitor.ts

@@ -0,0 +1,106 @@
+import _ from 'lodash'
+import { Connection, getConnections } from './connections'
+import { pendingPromise, wait } from './misc'
+import { ApiHandlers, SendListReadable } from './apiMiddleware'
+import Koa from 'koa'
+import { totalGot, totalInSpeed, totalOutSpeed, totalSent } from './throttler'
+import { getCurrentUsername } from './perm'
+
+const apis: ApiHandlers = {
+
+    async disconnect({ ip, port, wait }) {
+        const match = _.matches({ ip, port })
+        const c = getConnections().find(c => match(getConnAddress(c)))
+        const waiter = pendingPromise<void>()
+        c?.socket.end(waiter.resolve)
+        if (wait)
+            await waiter
+        return { result: Boolean(c) }
+    },
+
+    get_connections({}, ctx) {
+        const list = new SendListReadable({ addAtStart: getConnections().map(c => serializeConnection(c)) })
+        type Change = Partial<Omit<Connection,'ip'>>
+        const throttledUpdate = _.throttle(update, 1000/20) // try to avoid clogging with updates
+        const state = Symbol('state') // undefined=added, Timeout=add-pending, false=removed
+        return list.events(ctx, {
+            connection(conn: Connection) {
+                conn[state] = setTimeout(() => add(conn), 100)
+            },
+            connectionClosed(conn: Connection) {
+                if (cancel(conn)) return
+                list.remove(serializeConnection(conn, true))
+                conn[state] = false
+            },
+            connectionUpdated(conn: Connection, change: Change) {
+                if (!change.ctx)
+                    return throttledUpdate(conn, change)
+
+                Object.assign(change, fromCtx(change.ctx))
+                change.ctx = undefined
+                if (!add(conn))
+                    throttledUpdate(conn, change)
+            },
+        })
+
+        function add(conn: Connection) {
+            if (!cancel(conn)) return
+            list.add(serializeConnection(conn))
+            return true
+        }
+
+        function cancel(conn: Connection) {
+            if (!conn[state]) return
+            clearTimeout(conn[state])
+            conn[state] = undefined
+            return true
+        }
+
+        function update(conn: Connection, change: Change) {
+            if (conn[state] === false) return
+            list.update(serializeConnection(conn, true), change)
+        }
+
+        function serializeConnection(conn: Connection, minimal?:true) {
+            const { socket, started, secure } = conn
+            return Object.assign(getConnAddress(conn), !minimal && {
+                v: (socket.remoteFamily?.endsWith('6') ? 6 : 4),
+                got: socket.bytesRead,
+                sent: socket.bytesWritten,
+                started,
+                secure: (secure || undefined) as boolean|undefined, // undefined will save some space once json-ed
+                ...fromCtx(conn.ctx),
+            })
+        }
+
+        function fromCtx(ctx?: Koa.Context) {
+            return ctx && {
+                user: getCurrentUsername(ctx),
+                archive: ctx.state.archive,
+                path: (ctx.fileSource || ctx.state.archive) && ctx.path  // only for downloading files
+            }
+        }
+    },
+
+    async *get_connection_stats() {
+        while (1) {
+            yield {
+                outSpeed: totalOutSpeed,
+                inSpeed: totalInSpeed,
+                got: totalGot,
+                sent: totalSent,
+                connections: getConnections().length
+            }
+            await wait(1000)
+        }
+    },
+}
+
+export default apis
+
+function getConnAddress(conn: Connection) {
+    return {
+        ip: conn.ip,
+        port: conn.socket.remotePort,
+    }
+}

package/src/api.plugins.ts

@@ -0,0 +1,139 @@
+import {
+    AvailablePlugin,
+    enablePlugins,
+    getAvailablePlugins,
+    getPluginConfigFields,
+    mapPlugins,
+    Plugin, pluginsConfig,
+    PATH as PLUGINS_PATH, isPluginRunning, enablePlugin, getPluginInfo, setPluginConfig
+} from './plugins'
+import _ from 'lodash'
+import assert from 'assert'
+import { objSameKeys, onOff, wait } from './misc'
+import { ApiHandlers, SendListReadable } from './apiMiddleware'
+import events from './events'
+import { rm } from 'fs/promises'
+import { downloadPlugin, getFolder2repo, getRepoInfo, readOnlinePlugin, searchPlugins } from './github'
+
+const apis: ApiHandlers = {
+
+    get_plugins({}, ctx) {
+        const list = new SendListReadable({ addAtStart: [ ...mapPlugins(serialize), ...getAvailablePlugins() ] })
+        return list.events(ctx, {
+            pluginInstalled: p => list.add(serialize(p)),
+            'pluginStarted pluginStopped pluginUpdated': p => {
+                const { id, ...rest } = serialize(p)
+                list.update({ id }, rest)
+            },
+            pluginUninstalled: id => list.remove({ id }),
+        })
+
+        function serialize(p: Readonly<Plugin> | AvailablePlugin) {
+            const o = 'getData' in p ? Object.assign(_.pick(p, ['id','started']), p.getData())
+                    : { ...p } // _.defaults mutates object, and we don't want that
+            return _.defaults(o, { started: null, badApi: null }) // nulls should be used to be sure to overwrite previous values,
+        }
+    },
+
+    async get_plugin_updates() {
+        const list = new SendListReadable()
+        setTimeout(async () => {
+            for (const [folder, repo] of Object.entries(getFolder2repo()))
+                try {
+                    if (!repo) continue
+                    const online = await readOnlinePlugin(await getRepoInfo(repo))
+                    if (!online.apiRequired || online.badApi) continue
+                    const disk = getPluginInfo(folder)
+                    if (online.version! > disk.version)
+                        list.add(online)
+                }
+                catch (err:any) {
+                    list.error(err.code || err.message)
+                }
+            list.close()
+        })
+        return list
+    },
+
+    async set_plugin({ id, enabled, config }) {
+        assert(id, 'id')
+        if (enabled !== undefined)
+            enablePlugin(id, enabled)
+        if (config)
+            setPluginConfig(id, config)
+        return {}
+    },
+
+    async get_plugin({ id }) {
+        return {
+            enabled: enablePlugins.get().includes(id),
+            config: {
+                ...objSameKeys(getPluginConfigFields(id) ||{}, v => v?.defaultValue),
+                ...pluginsConfig.get()[id]
+            }
+        }
+    },
+
+    search_online_plugins({ text }, ctx) {
+        return new SendListReadable({
+            async doAtStart(list) {
+                try {
+                    const folder2repo = getFolder2repo()
+                    for await (const pl of searchPlugins(text)) {
+                        const repo = pl.id
+                        const folder = _.findKey(folder2repo, x => x === repo)
+                        const installed = folder && getPluginInfo(folder)
+                        Object.assign(pl, {
+                            installed: _.includes(folder2repo, repo),
+                            update: installed && installed.version < pl.version!,
+                        })
+                        list.add(pl)
+                        // watch for events about this plugin, until this request is closed
+                        ctx.req.on('close', onOff(events, {
+                            pluginInstalled: p => {
+                                if (p.repo === repo)
+                                    list.update({ id: repo }, { installed: true })
+                            },
+                            pluginUninstalled: folder => {
+                                if (repo === getFolder2repo()[folder])
+                                    list.update({ id: repo }, { installed: false })
+                            },
+                            pluginUpdated: p => {
+                                if (p.repo === repo)
+                                    list.update({ id: repo }, { update: p.version < pl.version! })
+                            },
+                            ['pluginDownload_' + repo](status) {
+                                list.update({ id: repo }, { downloading: status ?? null })
+                            }
+                        }))
+                    }
+                } catch (err: any) {
+                    list.error(err.code || err.message)
+                }
+                list.ready()
+            }
+        })
+    },
+
+    async download_plugin(pl) {
+        const res = await downloadPlugin(pl.id, pl.branch)
+        return typeof res === 'string' ? getPluginInfo(res) : res
+    },
+
+    async update_plugin(pl) {
+        await downloadPlugin(pl.id, pl.branch, true)
+        return {}
+    },
+
+    async uninstall_plugin({ id }) {
+        while (isPluginRunning(id)) {
+            enablePlugin(id, false)
+            await wait(500)
+        }
+        await rm(PLUGINS_PATH + '/' + id,  { recursive: true, force: true })
+        return {}
+    }
+
+}
+
+export default apis

package/src/api.vfs.ts

@@ -0,0 +1,182 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { getNodeName, nodeIsDirectory, saveVfs, urlToNode, vfs, VfsNode } from './vfs'
+import _ from 'lodash'
+import { stat } from 'fs/promises'
+import { ApiError, ApiHandlers } from './apiMiddleware'
+import { dirname, join, resolve } from 'path'
+import { dirStream, isWindowsDrive, objSameKeys } from './misc'
+import { exec } from 'child_process'
+import { promisify } from 'util'
+import { FORBIDDEN, IS_WINDOWS } from './const'
+import { isMatch } from 'micromatch'
+
+type VfsAdmin = {
+    type?: string,
+    size?: number,
+    ctime?: Date,
+    mtime?: Date,
+    website?: true,
+    children?: VfsAdmin[]
+} & Omit<VfsNode, 'type' | 'children'>
+
+// to manipulate the tree we need the original node
+async function urlToNodeOriginal(uri: string) {
+    const n = await urlToNode(uri)
+    return n?.isTemp ? n.original : n
+}
+
+const apis: ApiHandlers = {
+
+    async get_vfs() {
+        return { root: vfs && await recur(vfs) }
+
+        async function recur(node: VfsNode): Promise<VfsAdmin> {
+            const dir = await nodeIsDirectory(node)
+            const stats: Pick<VfsAdmin, 'size' | 'ctime' | 'mtime'> = {}
+            try {
+                if (!dir)
+                    Object.assign(stats, _.pick(await stat(node.source!), ['size', 'ctime', 'mtime']))
+            }
+            catch {
+                stats.size = -1
+            }
+            if (stats && Number(stats.mtime) === Number(stats.ctime))
+                delete stats.mtime
+            const isRoot = node === vfs
+            return {
+                ...stats,
+                ...node,
+                website: dir && node.source && await stat(join(node.source, 'index.html')).then(() => true, () => undefined)
+                    || undefined,
+                name: isRoot ? undefined : getNodeName(node),
+                type: dir ? 'folder' : undefined,
+                children: node.children && await Promise.all(node.children.map(recur)),
+            }
+        }
+    },
+
+    async set_vfs({ uri, props }) {
+        const n = await urlToNodeOriginal(uri)
+        if (!n)
+            return new ApiError(404, 'path not found')
+        props = pickProps(props, ['name','source','can_see','can_read','masks','default'])
+        props = objSameKeys(props, v => v === null ? undefined : v) // null is a way to serialize undefined, that will restore default values
+        if (props.masks && typeof props.masks !== 'object')
+            delete props.masks
+        Object.assign(n, props)
+        if (getNodeName(_.omit(n, ['name'])) === n.name)  // name only if necessary
+            n.name = undefined
+        await saveVfs()
+        return n
+    },
+
+    async add_vfs({ under, source, name }) {
+        const n = under ? await urlToNodeOriginal(under) : vfs
+        if (!n)
+            return new ApiError(404, 'invalid under')
+        if (n.isTemp || !await nodeIsDirectory(n))
+            return new ApiError(FORBIDDEN, 'invalid under')
+        if (isWindowsDrive(source))
+            source += '\\' // slash must be included, otherwise it will refer to the cwd of that drive
+        const a = n.children || (n.children = [])
+        if (source && a.find(x => x.source === source))
+            return new ApiError(409, 'already present')
+        a.unshift({ source, name })
+        await saveVfs()
+        return {}
+    },
+
+    async del_vfs({ uris }) {
+        if (!uris || !Array.isArray(uris))
+            return new ApiError(400, 'invalid uris')
+        return {
+            errors: await Promise.all(uris.map(async uri => {
+                if (typeof uri !== 'string')
+                    return 400
+                const node = await urlToNodeOriginal(uri)
+                if (!node)
+                    return 404
+                const parent = dirname(uri)
+                const parentNode = await urlToNodeOriginal(parent)
+                if (!parentNode)
+                    return FORBIDDEN
+                const { children } = parentNode
+                if (!children) // shouldn't happen
+                    return 500
+                const idx = children.indexOf(node)
+                children.splice(idx, 1)
+                saveVfs()
+                return 0 // error code 0 is OK
+            }))
+        }
+    },
+
+    get_cwd() {
+        return { path: process.cwd() }
+    },
+
+    async resolve_path({ path, closestFolder }) {
+        path = resolve(path)
+        if (closestFolder)
+            while (path && !await stat(path).then(x => x.isDirectory(), () => 0))
+                path = dirname(path)
+        return { path }
+    },
+
+    async *ls({ path, files=true, fileMask }, ctx) {
+        if (!path && IS_WINDOWS) {
+            try {
+                for (const n of await getDrives())
+                    yield { add: { n, k: 'd' } }
+            }
+            catch(error) {
+                console.debug(error)
+            }
+            return
+        }
+        try {
+            path = isWindowsDrive(path) ? path + '\\' : resolve(path || '/')
+            for await (const name of dirStream(path)) {
+                if (ctx.req.aborted)
+                    return
+                try {
+                    const stats = await stat(join(path, name))
+                    if (stats.isFile())
+                        if (!files || fileMask && !isMatch(name, fileMask))
+                            continue
+                    yield {
+                        add: {
+                            n: name,
+                            s: stats.size,
+                            c: stats.ctime,
+                            m: stats.mtime,
+                            k: stats.isDirectory() ? 'd' : undefined,
+                        }
+                    }
+                }
+                catch {} // just ignore entries we can't stat
+            }
+        } catch (e: any) {
+            yield { error: e.code || e.message || String(e) }
+        }
+    }
+
+}
+
+export default apis
+
+// pick only selected props, and consider null and empty string as undefined
+function pickProps(o: any, keys: string[]) {
+    const ret: any = {}
+    if (o && typeof o === 'object')
+        for (const k of keys)
+            if (k in o)
+                ret[k] = o[k] === null || o[k] === '' ? undefined : o[k]
+    return ret
+}
+
+async function getDrives() {
+    const { stdout } = await promisify(exec)('wmic logicaldisk get name')
+    return stdout.split('\n').slice(1).map(x => x.trim()).filter(Boolean)
+}

package/src/apiMiddleware.ts

@@ -0,0 +1,124 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import Koa from 'koa'
+import createSSE from './sse'
+import { Readable } from 'stream'
+import { asyncGeneratorToReadable, onOff } from './misc'
+import events from './events'
+import { UNAUTHORIZED } from './const'
+import _, { DebouncedFunc } from 'lodash'
+
+export class ApiError extends Error {
+    constructor(public status:number, message?:string | Error) {
+        super(typeof message === 'string' ? message : message?.message)
+    }
+}
+type ApiHandlerResult = Record<string,any> | ApiError | Readable | AsyncGenerator<any>
+export type ApiHandler = (params:any, ctx:Koa.Context) => ApiHandlerResult | Promise<ApiHandlerResult>
+export type ApiHandlers = Record<string, ApiHandler>
+
+export function apiMiddleware(apis: ApiHandlers) : Koa.Middleware {
+    return async (ctx) => {
+        const { params } = ctx
+        console.debug('API', ctx.method, ctx.path, { ...params })
+        if (!apis.hasOwnProperty(ctx.path)) {
+            ctx.body = 'invalid api'
+            return ctx.status = 404
+        }
+        const csrf = ctx.cookies.get('csrf')
+        // we don't rely on SameSite cookie option because it's https-only
+        let res = csrf && csrf !== params.csrf ? new ApiError(UNAUTHORIZED, 'csrf')
+            : await apis[ctx.path](params || {}, ctx)
+        if (isAsyncGenerator(res))
+            res = asyncGeneratorToReadable(res)
+        if (res instanceof Readable) { // Readable, we'll go SSE-mode
+            res.pipe(createSSE(ctx))
+            const stillRes = res // satisfy ts
+            ctx.req.on('close', () => // by closing the generated stream, creator of the stream will know the request is over without having to access anything else
+                stillRes.destroy())
+            return
+        }
+        if (res instanceof ApiError) {
+            ctx.body = res.message
+            return ctx.status = res.status
+        }
+        if (res instanceof Error) { // generic exception
+            ctx.body = String(res)
+            return ctx.status = 400
+        }
+        ctx.body = res
+    }
+}
+
+function isAsyncGenerator(x: any): x is AsyncGenerator {
+    return typeof (x as AsyncGenerator)?.next === 'function'
+}
+
+// offer an api for a generic dynamic list. Suitable to be the result of an api.
+type SendListFunc<T> = (list:SendListReadable<T>) => void
+export class SendListReadable<T> extends Readable {
+    protected lastError: string | number | undefined
+    protected buffer: any[] = []
+    protected processBuffer: DebouncedFunc<any>
+    constructor({ addAtStart, doAtStart, bufferTime }:{ bufferTime?: number, addAtStart?: T[], doAtStart?: SendListFunc<T> }={}) {
+        super({ objectMode: true, read(){} })
+        if (!bufferTime)
+            bufferTime = 100
+        this.processBuffer = _.debounce(() => {
+            this.push(this.buffer)
+            this.buffer = []
+        }, bufferTime, { maxWait: bufferTime })
+        this.on('end', () =>
+            this.destroy())
+        if (doAtStart)
+            setTimeout(() => doAtStart(this)) // work later, when list object has been received by Koa
+        if (addAtStart) {
+            for (const x of addAtStart)
+                this.add(x)
+            this.ready()
+        }
+    }
+    protected _push(rec: any) {
+        this.buffer.push(rec)
+        if (this.buffer.length > 10_000) // hard limit
+            this.processBuffer.flush()
+        else
+            this.processBuffer()
+    }
+    add(rec: T | T[]) {
+        this._push({ add: rec })
+    }
+    remove(key: Partial<T>) {
+        this._push({ remove: [key] })
+    }
+    update(search: Partial<T>, change: Partial<T>) {
+        this._push({ update:[{ search, change }] })
+    }
+    ready() { // useful to indicate the end of an initial phase, but we leave open for updates
+        this._push('ready')
+    }
+    custom(data: any) {
+        this._push(data)
+    }
+    error(msg: NonNullable<typeof this.lastError>, close=false) {
+        this._push({ error: msg })
+        this.lastError = msg
+        if (close)
+            this.close()
+    }
+    getLastError() {
+        return this.lastError
+    }
+    close() {
+        this.processBuffer.flush()
+        this.push(null)
+    }
+    events(ctx: Koa.Context, eventMap: Parameters<typeof onOff>[1]) {
+        const off = onOff(events, eventMap)
+        ctx.res.once('close', off)
+        return this
+    }
+    isClosed() {
+        return this.destroyed
+    }
+}

package/src/block.ts

@@ -0,0 +1,35 @@
+import { defineConfig } from './config'
+import { getConnections, normalizeIp } from './connections'
+import { onlyTruthy, with_ } from './misc'
+import cidr from 'cidr-tools'
+import _ from 'lodash'
+import { Socket } from 'net'
+
+defineConfig<string[]>('block', []).sub(rules => {
+    compileBlock(rules)
+    for (const { socket, ip } of getConnections())
+        applyBlock(socket, ip)
+})
+
+type BlockFun = (x: string) => boolean
+let blockFunctions: BlockFun[] = [] // "compiled" versions of the rules in config.block
+
+function compileBlock(rules: any) {
+    blockFunctions = !Array.isArray(rules) ? []
+        : onlyTruthy(rules.map(rule => !rule ? null
+            : with_(rule.ip, ip => typeof ip !== 'string' ? null
+                : ip.includes('/') ? x => cidr.contains(ip, x)
+                    : ip.includes('*') ? with_(ipMask2regExp(ip), re => x => re.test(x) )
+                        : x => x === ip
+            )
+        ))
+
+    function ipMask2regExp(ipMask: string) {
+        return new RegExp(_.escapeRegExp(ipMask).replace(/\\\*/g, '.*'))
+    }
+}
+
+export function applyBlock(socket: Socket, ip=normalizeIp(socket.remoteAddress||'')) {
+    if (ip && blockFunctions.find(rule => rule(ip)))
+        return socket.destroy()
+}