hfs 0.26.7 → 0.26.8

package/src/ThrottledStream.ts

@@ -0,0 +1,98 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { Transform, TransformCallback } from 'stream'
+import { TokenBucket } from 'limiter'
+
+// throttled stream
+export class ThrottledStream extends Transform {
+
+    private sent: number = 0
+    private lastSpeed: number = 0
+    private lastSpeedTime = Date.now()
+    private totalSent: number = 0 // total sent over connection, since connection can be re-used for multiple requests
+
+    constructor(private group: ThrottleGroup, copyStats?: ThrottledStream) {
+        super()
+        if (!copyStats) return
+        this.sent = copyStats.sent
+        this.totalSent = copyStats.totalSent
+        this.lastSpeedTime = copyStats.lastSpeedTime
+        this.lastSpeed = copyStats.lastSpeed
+    }
+
+    async _transform(chunk: any, encoding: BufferEncoding, done: TransformCallback) {
+        let pos = 0
+        while (1) {
+            let n = this.group.suggestChunkSize()
+            const slice = chunk.slice(pos, pos + n)
+            n = slice.length
+            if (!n) // we're done here
+                return done()
+            try {
+                await this.group.consume(n)
+                this.push(slice)
+                this.sent += n
+                this.totalSent += n
+                pos += n
+                this.emit('sent', n)
+            } catch (e) {
+                done(e as Error)
+                return
+            }
+        }
+    }
+
+    // @return kBs
+    getSpeed(): number {
+        const now = Date.now()
+        const past = now - this.lastSpeedTime
+        if (past >= 1000) { // recalculate?
+            this.lastSpeedTime = now
+            this.lastSpeed = this.sent / past
+            this.sent = 0
+        }
+        return this.lastSpeed
+    }
+
+    getBytesSent() {
+        return this.totalSent
+    }
+}
+
+export class ThrottleGroup {
+
+    private bucket: TokenBucket
+
+    constructor(kBs: number, private parent?: ThrottleGroup) {
+        this.bucket = this.updateLimit(kBs) // assignment is redundant and yet the best way I've found to shut up typescript
+    }
+
+    // @return kBs
+    getLimit() {
+        return this.bucket.bucketSize / 1000
+    }
+
+    updateLimit(kBs: number) {
+        if (kBs < 0)
+            throw new Error('invalid bytesPerSecond')
+        kBs *= 1000
+        return this.bucket = new TokenBucket({
+            bucketSize: kBs,
+            tokensPerInterval: kBs,
+            interval: 'second',
+        })
+    }
+
+    suggestChunkSize() {
+        let b: TokenBucket | undefined = this.bucket
+        b.parentBucket = this.parent?.bucket
+        let min = b.bucketSize
+        while (b = b.parentBucket)
+            min = Math.min(min, b.bucketSize)
+        return min / 10
+    }
+
+    consume(n: number) {
+        return this.bucket.removeTokens(n)
+    }
+}

package/src/adminApis.ts

@@ -0,0 +1,161 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { ApiError, ApiHandlers, SendListReadable } from './apiMiddleware'
+import { defineConfig, getWholeConfig, setConfig } from './config'
+import { getStatus, getUrls, httpsPortCfg, portCfg } from './listen'
+import {
+    API_VERSION,
+    BUILD_TIMESTAMP,
+    COMPATIBLE_API_VERSION,
+    FORBIDDEN,
+    HFS_STARTED,
+    IS_WINDOWS,
+    UNAUTHORIZED,
+    VERSION
+} from './const'
+import vfsApis from './api.vfs'
+import accountsApis from './api.accounts'
+import pluginsApis from './api.plugins'
+import monitorApis from './api.monitor'
+import { getConnections } from './connections'
+import { debounceAsync, isLocalHost, onOff, wait } from './misc'
+import _ from 'lodash'
+import events from './events'
+import { getFromAccount } from './perm'
+import Koa from 'koa'
+import { getProxyDetected } from './middlewares'
+import { writeFile } from 'fs/promises'
+import { createReadStream } from 'fs'
+import * as readline from 'readline'
+import { loggers } from './log'
+import { execFile } from 'child_process'
+import { promisify } from 'util'
+
+export const adminApis: ApiHandlers = {
+
+    ...vfsApis,
+    ...accountsApis,
+    ...pluginsApis,
+    ...monitorApis,
+
+    async set_config({ values: v }) {
+        if (v) {
+            const st = getStatus()
+            const noHttp = (v.port ?? portCfg.get()) < 0 || !st.httpSrv.listening
+            const noHttps = (v.https_port ?? httpsPortCfg.get()) < 0 || !st.httpsSrv.listening
+            if (noHttp && noHttps)
+                return new ApiError(FORBIDDEN, "You cannot switch off both http and https ports")
+            await setConfig(v)
+        }
+        return {}
+    },
+
+    get_config: getWholeConfig,
+
+    async get_status() {
+        const st = getStatus()
+        return {
+            started: HFS_STARTED,
+            build: BUILD_TIMESTAMP,
+            version: VERSION,
+            apiVersion: API_VERSION,
+            compatibleApiVersion: COMPATIBLE_API_VERSION,
+            http: await serverStatus(st.httpSrv, portCfg.get()),
+            https: await serverStatus(st.httpsSrv, httpsPortCfg.get()),
+            urls: getUrls(),
+            proxyDetected: getProxyDetected(),
+            frpDetected: localhostAdmin.get() && !getProxyDetected()
+                && getConnections().every(isLocalHost)
+                && await frpDebounced(),
+        }
+
+        async function serverStatus(h: typeof st.httpSrv, configuredPort?: number) {
+            const busy = await h.busy
+            await wait(0) // simple trick to wait for also .error to be updated. If this trickery becomes necessary elsewhere, then we should make also error a Promise.
+            return {
+                ..._.pick(h, ['listening', 'error']),
+                busy,
+                port: (h?.address() as any)?.port || configuredPort,
+            }
+        }
+    },
+
+    async save_pem({ cert, private_key, name='self' }) {
+        if (!cert || !private_key)
+            return new ApiError(400)
+        const files = { cert: name + '.cert', private_key: name + '.key' }
+        await writeFile(files.private_key, private_key)
+        await writeFile(files.cert, cert)
+        return files
+    },
+
+    async get_log({ file='log' }, ctx) {
+        return new SendListReadable({
+            bufferTime: 10,
+            doAtStart(list) {
+                const logger = loggers.find(l => l.name === file)
+                if (!logger)
+                    return list.error(404, true)
+                const input = createReadStream(logger.path)
+                input.on('error', async (e: any) => {
+                    if (e.code === 'ENOENT') // ignore ENOENT, consider it an empty log
+                        return list.ready()
+                    list.error(e.code || e.message)
+                })
+                input.on('end', () =>
+                    list.ready())
+                input.on('ready', () => {
+                    readline.createInterface({ input }).on('line', line => {
+                        if (ctx.aborted)
+                            return input.close()
+                        const obj = parse(line)
+                        if (obj)
+                            list.add(obj)
+                    }).on('close', () => { // file is automatically closed, so we continue by events
+                        ctx.res.once('close', onOff(events, { // unsubscribe when connection is interrupted
+                            [logger.name](entry) {
+                                list.add(entry)
+                            }
+                        }))
+                    })
+                })
+            }
+        })
+
+        function parse(line: string) {
+            const m = /^(.+?) (.+?) (.+?) \[(.{11}):(.{14})] "(\w+) ([^"]+) HTTP\/\d.\d" (\d+) (-|\d+)/.exec(line)
+            if (!m) return
+            const [, ip, , user, date, time, method, uri, status, length] = m
+            return { // keep object format same as events emitted by the log module
+                ip,
+                user: user === '-' ? undefined : user,
+                ts: new Date(date + ' ' + time),
+                method,
+                uri,
+                status: Number(status),
+                length: length === '-' ? undefined : Number(length),
+            }
+        }
+    },
+}
+
+for (const k in adminApis) {
+    const was = adminApis[k]
+    adminApis[k] = (params, ctx) =>
+        ctxAdminAccess(ctx) ? was(params, ctx)
+            : new ApiError(UNAUTHORIZED)
+}
+
+export const localhostAdmin = defineConfig('localhost_admin', true)
+
+export function ctxAdminAccess(ctx: Koa.Context) {
+    return !ctx.state.proxiedFor // we consider localhost_admin only if no proxy is detected
+        && localhostAdmin.get() && isLocalHost(ctx)
+        || getFromAccount(ctx.state.account, a => a.admin)
+}
+
+const frpDebounced = debounceAsync(async () => {
+    if (!IS_WINDOWS) return false
+    const { stdout } = await promisify(execFile)('tasklist', ['/fi','imagename eq frpc.exe','/nh'])
+    return stdout.includes('frpc')
+})

package/src/api.accounts.ts

@@ -0,0 +1,78 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { changePasswordHelper, changeSrpHelper } from './api.helpers'
+import { ApiError, ApiHandlers } from './apiMiddleware'
+import {
+    Account,
+    accountCanLoginAdmin,
+    accountHasPassword,
+    addAccount,
+    delAccount,
+    getAccount,
+    getAccounts,
+    getCurrentUsername,
+    setAccount
+} from './perm'
+import _ from 'lodash'
+import { FORBIDDEN } from './const'
+
+function prepareAccount(ac: Account | undefined) {
+    return ac && {
+        ..._.omit(ac, ['password','hashed_password','srp']),
+        username: ac.username, // omit won't copy it because it's a hidden prop
+        hasPassword: accountHasPassword(ac),
+        adminActualAccess: accountCanLoginAdmin(ac),
+    }
+}
+
+const apis: ApiHandlers = {
+
+    get_usernames() {
+        return { list: Object.keys(getAccounts()) }
+    },
+
+    get_account({ username }, ctx) {
+        return prepareAccount(getAccount(username || getCurrentUsername(ctx)))
+            || new ApiError(404)
+    },
+
+    get_accounts() {
+        return { list: Object.values(getAccounts()).map(prepareAccount) }
+    },
+
+    get_admins() {
+        return { list: Object.values(getAccounts()).map(prepareAccount).filter(ac => ac?.adminActualAccess).map(ac => ac!.username) }
+    },
+
+    set_account({ username, changes }) {
+        const { admin } = changes
+        if (admin === null)
+            changes.admin = undefined
+        else if (admin !== undefined && typeof admin !== 'boolean')
+            return new ApiError(400, "invalid admin")
+        return setAccount(username, changes) ? {} : new ApiError(400)
+    },
+
+    add_account({ username, ...rest }) {
+        if (getAccount(username))
+            return new ApiError(FORBIDDEN)
+        if (!addAccount(username, rest))
+            return new ApiError(400)
+        return {}
+    },
+
+    del_account({ username }) {
+        return delAccount(username) ? {} : new ApiError(400)
+    },
+
+    async change_password_others({ username, newPassword }) {
+        return changePasswordHelper(getAccount(username), newPassword)
+    },
+
+    async change_srp_others({ username, salt, verifier }) {
+        return changeSrpHelper(getAccount(username), salt, verifier)
+    }
+
+}
+
+export default apis

package/src/api.auth.ts

@@ -0,0 +1,131 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { Account, getAccount, getCurrentUsername } from './perm'
+import { verifyPassword } from './crypt'
+import { ApiError, ApiHandler } from './apiMiddleware'
+import { SRPParameters, SRPRoutines, SRPServerSession, SRPServerSessionStep1 } from 'tssrp6a'
+import { ADMIN_URI, SESSION_DURATION, UNAUTHORIZED } from './const'
+import { randomId } from './misc'
+import Koa from 'koa'
+import { changeSrpHelper, changePasswordHelper } from './api.helpers'
+import { ctxAdminAccess } from './adminApis'
+import { prepareState } from './middlewares'
+
+const srp6aNimbusRoutines = new SRPRoutines(new SRPParameters())
+const ongoingLogins:Record<string,SRPServerSessionStep1> = {} // store data that doesn't fit session object
+
+// centralized log-in state
+async function loggedIn(ctx:Koa.Context, username: string | false) {
+    const s = ctx.session
+    if (!s)
+        return ctx.throw(500,'session')
+    if (username === false) {
+        delete s.username
+        ctx.cookies.set('csrf', '')
+        return
+    }
+    s.username = username
+    await prepareState(ctx, async ()=>{}) // updating the state is necessary to send complete session data so that frontend shows admin button
+    delete s.login
+    ctx.cookies.set('csrf', randomId(), { signed:false, httpOnly: false })
+}
+
+function makeExp() {
+    return { exp: new Date(Date.now() + SESSION_DURATION) }
+}
+
+export const login: ApiHandler = async ({ username, password }, ctx) => {
+    if (!username || !password) // some validation
+        return new ApiError(400)
+    username = username.toLocaleLowerCase() // normalize username, to be case-insensitive
+    const acc = getAccount(username)
+    if (!acc)
+        return new ApiError(UNAUTHORIZED)
+    if (!acc.hashed_password)
+        return new ApiError(406)
+    if (!await verifyPassword(acc.hashed_password, password))
+        return new ApiError(UNAUTHORIZED)
+    if (!ctx.session)
+        return new ApiError(500)
+    await loggedIn(ctx, username)
+    return { ...makeExp(), redirect: acc.redirect }
+}
+
+export const loginSrp1: ApiHandler = async ({ username }, ctx) => {
+    if (!username)
+        return new ApiError(400)
+    username = username.toLocaleLowerCase()
+    const account = getAccount(username)
+    if (!ctx.session)
+        return new ApiError(500)
+    if (!account) // TODO simulate fake account to prevent knowing valid usernames
+        return new ApiError(UNAUTHORIZED)
+    try {
+        const { step1, ...rest } = await srpStep1(account)
+        const sid = Math.random()
+        ongoingLogins[sid] = step1
+        setTimeout(()=> delete ongoingLogins[sid], 60_000)
+        ctx.session.login = { username, sid }
+        return rest
+    }
+    catch (code: any) {
+        return new ApiError(code)
+    }
+}
+
+export async function srpStep1(account: Account) {
+    if (!account.srp)
+        throw 406 // unacceptable
+    const [salt, verifier] = account.srp.split('|')
+    const srpSession = new SRPServerSession(srp6aNimbusRoutines)
+    const step1 = await srpSession.step1(account.username, BigInt(salt), BigInt(verifier))
+    return { step1, salt, pubKey: String(step1.B) } // cast to string cause bigint can't be jsonized
+}
+
+export const loginSrp2: ApiHandler = async ({ pubKey, proof }, ctx) => {
+    if (!ctx.session)
+        return new ApiError(500)
+    if (!ctx.session.login)
+        return new ApiError(409)
+    const { username, sid } = ctx.session.login
+    const step1 = ongoingLogins[sid]
+    try {
+        const M2 = await step1.step2(BigInt(pubKey), BigInt(proof))
+        await loggedIn(ctx, username)
+        return {
+            proof: String(M2),
+            redirect: ctx.state.account?.redirect,
+            ...await refresh_session({},ctx)
+        }
+    }
+    catch(e) {
+        return new ApiError(UNAUTHORIZED, String(e))
+    }
+    finally {
+        delete ongoingLogins[sid]
+    }
+}
+
+export const logout: ApiHandler = async ({}, ctx) => {
+    if (!ctx.session)
+        return new ApiError(500)
+    loggedIn(ctx, false)
+    // 401 is a convenient code for OK: the browser clears a possible http authentication (hopefully), and Admin automatically triggers login dialog
+    return new ApiError(401)
+}
+
+export const refresh_session: ApiHandler = async ({}, ctx) => {
+    return !ctx.session ? new ApiError(500) : {
+        username: getCurrentUsername(ctx),
+        adminUrl: ctxAdminAccess(ctx) ? ADMIN_URI : undefined,
+        ...makeExp(),
+    }
+}
+
+export const change_password: ApiHandler = async ({ newPassword }, ctx) => {
+    return changePasswordHelper(ctx.state.account, newPassword)
+}
+
+export const change_srp: ApiHandler = async ({ salt, verifier }, ctx) => {
+    return changeSrpHelper(ctx.state.account, salt, verifier)
+}

package/src/api.file_list.ts

@@ -0,0 +1,102 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { cantReadStatusCode, getNodeName, hasPermission, nodeIsDirectory, urlToNode, VfsNode, walkNode } from './vfs'
+import { ApiError, ApiHandler, SendListReadable } from './apiMiddleware'
+import { stat } from 'fs/promises'
+import { mapPlugins } from './plugins'
+import { asyncGeneratorToArray, dirTraversal, pattern2filter } from './misc'
+import _ from 'lodash'
+
+export const file_list: ApiHandler = async ({ path, offset, limit, search, omit, sse }, ctx) => {
+    let node = await urlToNode(path || '/', ctx)
+    const list = new SendListReadable()
+    if (!node)
+        return fail(404)
+    if (!hasPermission(node,'can_read',ctx))
+        return fail(cantReadStatusCode(node))
+    if (dirTraversal(search))
+        return fail(418)
+    if (node.default)
+        return (sse ? list.custom : _.identity)({ redirect: path }) // sse will wrap the object in a 'custom' message, otherwise we plainly return the object
+    if (!await nodeIsDirectory(node))
+        return fail(405) // method not allowed on target
+    offset = Number(offset)
+    limit = Number(limit)
+    const filter = pattern2filter(search)
+    const walker = walkNode(node, ctx, search ? Infinity : 0)
+    const onDirEntryHandlers = mapPlugins(plug => plug.onDirEntry)
+    if (!sse)
+        return { list: await asyncGeneratorToArray(produceEntries()) }
+    setTimeout(async () => {
+        for await (const entry of produceEntries())
+            list.add(entry)
+        list.close()
+    })
+    return list
+
+    function fail(code: any) {
+        if (!sse)
+            return new ApiError(code)
+        list.error(code)
+        list.close()
+        return list
+    }
+
+    async function* produceEntries() {
+        for await (const sub of walker) {
+            if (ctx.aborted) break
+            if (!filter(getNodeName(sub)))
+                continue
+            const entry = await nodeToDirEntry(sub)
+            if (!entry)
+                continue
+            const cbParams = { entry, ctx, listPath:path, node:sub }
+            try {
+                if (onDirEntryHandlers.some(cb => cb(cbParams) === false))
+                    continue
+            }
+            catch(e) {
+                console.log("a plugin with onDirEntry is causing problems:", e)
+            }
+            if (offset) {
+                --offset
+                continue
+            }
+            if (omit) {
+                if (omit !== 'c')
+                    ctx.throw(400, 'omit')
+                if (!entry.m)
+                    entry.m = entry.c
+                delete entry.c
+            }
+            yield entry
+            if (limit && !--limit)
+                break
+        }
+    }
+}
+
+export interface DirEntry { n:string, s?:number, m?:Date, c?:Date }
+
+async function nodeToDirEntry(node: VfsNode): Promise<DirEntry | null> {
+    let { source, default:def } = node
+    const name = getNodeName(node)
+    if (!source)
+        return name ? { n: name + '/' } : null
+    if (def)
+        return { n: name }
+    try {
+        const st = await stat(source)
+        const folder = st.isDirectory()
+        const { ctime, mtime } = st
+        return {
+            n: name + (folder ? '/' : ''),
+            c: ctime,
+            m: Math.abs(+mtime-+ctime) < 1000 ? undefined : mtime,
+            s: folder ? undefined : st.size,
+        }
+    }
+    catch {
+        return null
+    }
+}

package/src/api.helpers.ts

@@ -0,0 +1,30 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { Account, allowClearTextLogin, saveSrpInfo, updateAccount } from './perm'
+import { ApiError } from './apiMiddleware'
+import { UNAUTHORIZED } from './const'
+
+export async function changePasswordHelper(account: Account | undefined, newPassword: string) {
+    if (!newPassword) // clear text version
+        return Error('missing parameters')
+    if (!account)
+        return new ApiError(UNAUTHORIZED)
+    await updateAccount(account, account => {
+        account.password = newPassword
+    })
+    return {}
+}
+
+export async function changeSrpHelper(account: Account | undefined, salt: string, verifier: string) {
+    if (allowClearTextLogin.get())
+        return new ApiError(406)
+    if (!salt || !verifier)
+        return Error('missing parameters')
+    if (!account)
+        return new ApiError(UNAUTHORIZED)
+    await updateAccount(account, account => {
+        saveSrpInfo(account, salt, verifier)
+        delete account.hashed_password // remove leftovers
+    })
+    return {}
+}