hfs 0.26.7 → 0.26.8

package/src/serveFile.ts

@@ -0,0 +1,104 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import Koa from 'koa'
+import { createReadStream, stat } from 'fs'
+import { FORBIDDEN, METHOD_NOT_ALLOWED, NO_CONTENT } from './const'
+import { getNodeName, MIME_AUTO, VfsNode } from './vfs'
+import mimetypes from 'mime-types'
+import { defineConfig } from './config'
+import { isMatch } from 'micromatch'
+import _ from 'lodash'
+import path from 'path'
+import { promisify } from 'util'
+
+const allowedReferer = defineConfig('allowed_referer', '')
+
+export function serveFileNode(node: VfsNode) : Koa.Middleware {
+    const { source, mime } = node
+    const name = getNodeName(node)
+    const mimeString = typeof mime === 'string' ? mime
+        : _.find(mime, (val,mask) => isMatch(name, mask))
+    return (ctx, next) => {
+       const allowed = allowedReferer.get()
+        if (allowed) {
+            const ref = /\/\/([^:/]+)/.exec(ctx.get('referer'))?.[1] // extract host from url
+            if (ref && ref !== host() // automatic accept if referer is basically the hosting domain
+            && !isMatch(ref, allowed))
+                return ctx.status = FORBIDDEN
+
+            function host() {
+                const s = ctx.get('host')
+                return s[0] === '[' ? s.slice(1, s.indexOf(']')) : s?.split(':')[0]
+            }
+        }
+
+        ctx.vfsNode = node // useful to tell service files from files shared by the user
+        return serveFile(source||'', mimeString)(ctx, next)
+    }
+}
+
+const mimeCfg = defineConfig<Record<string,string>>('mime', { '*': 'auto' })
+
+export function serveFile(source:string, mime?:string, content?: string | Buffer) : Koa.Middleware {
+    return async (ctx) => {
+        if (!source)
+            return
+        const fn = path.basename(source)
+        mime = mime ?? _.find(mimeCfg.get(), (v,k) => k>'' && isMatch(fn, k)) // isMatch throws on an empty string
+        if (mime === MIME_AUTO)
+            mime = mimetypes.lookup(source) || ''
+        if (mime)
+            ctx.type = mime
+        if (ctx.method === 'OPTIONS') {
+            ctx.status = NO_CONTENT
+            ctx.set({ Allow: 'OPTIONS, GET, HEAD' })
+            return
+        }
+        if (ctx.method !== 'GET')
+            return ctx.status = METHOD_NOT_ALLOWED
+        try {
+            const stats = await promisify(stat)(source) // using fs's function instead of fs/promises, because only the former is supported by pkg
+            ctx.set('Last-Modified', stats.mtime.toUTCString())
+            ctx.fileSource = source
+            ctx.status = 200
+            if (ctx.fresh)
+                return ctx.status = 304
+            if (content !== undefined)
+                return ctx.body = content
+            const range = getRange(ctx, stats.size)
+            ctx.body = createReadStream(source, range)
+        }
+        catch {
+            return ctx.status = 404
+        }
+    }
+}
+
+export function getRange(ctx: Koa.Context, totalSize: number) {
+    ctx.set('Accept-Ranges', 'bytes')
+    const { range } = ctx.request.header
+    if (!range) {
+        ctx.response.length = totalSize
+        return
+    }
+    const ranges = range.split('=')[1]
+    if (ranges.includes(','))
+        return ctx.throw(400, 'multi-range not supported')
+    let bytes = ranges?.split('-')
+    if (!bytes?.length)
+        return ctx.throw(400, 'bad range')
+    const max = totalSize - 1
+    const start = bytes[0] ? Number(bytes[0]) : Math.max(0, totalSize-Number(bytes[1])) // a negative start is relative to the end
+    const end = bytes[0] ? Number(bytes[1] || max) : max
+    // we don't support last-bytes without knowing max
+    if (isNaN(end) && isNaN(max) || end > max || start > max) {
+        ctx.status = 416
+        ctx.set('Content-Range', `bytes ${totalSize}`)
+        ctx.body = 'Requested Range Not Satisfiable'
+        return
+    }
+    ctx.status = 206
+    ctx.set('Content-Range', `bytes ${start}-${isNaN(end) ? '' : end}/${isNaN(totalSize) ? '*' : totalSize}`)
+    ctx.response.length = end - start + 1
+    return { start, end }
+}

package/src/serveGuiFiles.ts

@@ -0,0 +1,95 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import Koa from 'koa'
+import fs from 'fs/promises'
+import { METHOD_NOT_ALLOWED, NO_CONTENT, PLUGINS_PUB_URI } from './const'
+import { serveFile } from './serveFile'
+import { mapPlugins } from './plugins'
+import { refresh_session } from './api.auth'
+import { ApiError } from './apiMiddleware'
+import { join, extname } from 'path'
+import { getOrSet } from './misc'
+
+// in case of dev env we have our static files within the 'dist' folder'
+const DEV_STATIC = process.env.DEV ? 'dist/' : ''
+
+function serveStatic(uri: string): Koa.Middleware {
+    const folder = uri.slice(2,-1) // we know folder is very similar to uri
+    const cache: Record<string, Promise<string>> = {}
+    return async (ctx, next) => {
+        if(ctx.method === 'OPTIONS') {
+            ctx.status = NO_CONTENT
+            ctx.set({ Allow: 'OPTIONS, GET' })
+            return
+        }
+        if (ctx.method !== 'GET')
+            return ctx.status = METHOD_NOT_ALLOWED
+        const serveApp = shouldServeApp(ctx)
+        const fullPath = join(__dirname, '..', DEV_STATIC, folder, serveApp ? '/index.html': ctx.path)
+        const content = await getOrSet(cache, ctx.path, async () => {
+            const data = await fs.readFile(fullPath).catch(() => null)
+            return serveApp || !data ? data
+                : adjustBundlerLinks(ctx.path, uri, data)
+        })
+        if (content === null)
+            return ctx.status = 404
+        if (!serveApp)
+            return serveFile(fullPath, 'auto', content)(ctx, next)
+        // we don't cache the index as it's small and may prevent plugins change to apply
+        ctx.body = await treatIndex(ctx, String(content), uri)
+        ctx.type = 'html'
+        ctx.set('Cache-Control', 'no-store, no-cache, must-revalidate')
+    }
+}
+
+function shouldServeApp(ctx: Koa.Context) {
+    return ctx.state.serveApp ||= ctx.path.endsWith('/')
+}
+
+function adjustBundlerLinks(path: string, uri: string, data: string | Buffer) {
+    const ext = extname(path)
+    return ext && !ext.match(/\.(css|html|js|ts|scss)/) ? data
+        : String(data).replace(/((?:import | from )['"])\//g, `$1${uri}`)
+}
+
+async function treatIndex(ctx: Koa.Context, body: string, filesUri: string) {
+    const session = await refresh_session({}, ctx)
+    ctx.set('etag', '')
+    return body
+        .replace(/((?:src|href) *= *['"])\/?(?![a-z]+:\/\/)/g, '$1' + filesUri)
+        .replace('_HFS_SESSION_', session instanceof ApiError ? 'null' : JSON.stringify(session))
+        // replacing this text allow us to avoid injecting in frontends that don't support plugins. Don't use a <--comment--> or it will be removed by webpack
+        .replace('_HFS_PLUGINS_', pluginsInjection)
+}
+
+function serveProxied(port: string | undefined, uri: string) { // used for development only
+    if (!port)
+        return
+    console.debug('proxied on port', port)
+    let proxy: Koa.Middleware
+    import('koa-better-http-proxy').then(lib => // dynamic import to avoid having this in final distribution
+        proxy = lib.default('127.0.0.1:'+port, {
+            proxyReqPathResolver: (ctx) =>
+                shouldServeApp(ctx) ? '/' : ctx.path,
+            userResDecorator(res, data, ctx) {
+                return shouldServeApp(ctx) ? treatIndex(ctx, String(data), uri)
+                    : adjustBundlerLinks(ctx.path, uri, data)
+            }
+        }) )
+    return function() { //@ts-ignore
+        return proxy.apply(this,arguments)
+    }
+}
+
+function pluginsInjection() {
+    const css = mapPlugins((plug,k) =>
+        plug.frontend_css?.map(f => PLUGINS_PUB_URI + k + '/' + f)).flat().filter(Boolean)
+    const js = mapPlugins((plug,k) =>
+        plug.frontend_js?.map(f => PLUGINS_PUB_URI + k + '/' + f)).flat().filter(Boolean)
+    return css.map(uri => `\n<link rel='stylesheet' type='text/css' href='${uri}'/>`).join('')
+        + js.map(uri => `\n<script defer src='${uri}'></script>`).join('')
+}
+
+export function serveGuiFiles(proxyPort:string | undefined, uri:string) {
+    return serveProxied(proxyPort, uri) || serveStatic(uri)
+}

package/src/sse.ts

@@ -0,0 +1,29 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import Koa from 'koa'
+import { Transform } from 'stream'
+
+export default function createSSE(ctx: Koa.Context) {
+    const { socket } = ctx.req
+    socket.setTimeout(0)
+    socket.setNoDelay(true)
+    socket.setKeepAlive(true)
+    ctx.set({
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'X-Accel-Buffering': 'no', // avoid buffering when reverse-proxied through nginx
+    })
+    ctx.status = 200
+    return ctx.body = new Transform({
+        objectMode: true,
+        transform(chunk, encoding, cb) {
+            this.push(`data: ${JSON.stringify(chunk)}\n\n`)
+            cb()
+        },
+        flush(cb) {
+            this.push('data:\n\n')
+            cb()
+        }
+    })
+}

package/src/throttler.ts

@@ -0,0 +1,106 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { Readable } from 'stream'
+import Koa from 'koa'
+import { ThrottledStream, ThrottleGroup } from './ThrottledStream'
+import { defineConfig } from './config'
+import { getOrSet, isLocalHost } from './misc'
+import { Connection, updateConnection } from './connections'
+import _ from 'lodash'
+import events from './events'
+
+const mainThrottleGroup = new ThrottleGroup(Infinity)
+
+defineConfig('max_kbps', Infinity).sub(v =>
+    mainThrottleGroup.updateLimit(v))
+
+const ip2group: Record<string, {
+    count: number
+    group: ThrottleGroup
+    destroy: () => void
+}> = {}
+
+const SymThrStr = Symbol('stream')
+const SymTimeout = Symbol('timeout')
+
+const maxKbpsPerIp = defineConfig('max_kbps_per_ip', Infinity)
+
+export const throttler: Koa.Middleware = async (ctx, next) => {
+    await next()
+    const { body } = ctx
+    if (!body || !(body instanceof Readable))
+        return
+    // we wrap the stream also for unlimited connections to get speed and other features
+    const ipGroup = getOrSet(ip2group, ctx.ip, ()=> {
+        const doLimit = ctx.state.account?.ignore_limits || isLocalHost(ctx) ? undefined : true
+        const group = new ThrottleGroup(Infinity, doLimit && mainThrottleGroup)
+
+        const unsub = doLimit && maxKbpsPerIp.sub(v =>
+            group.updateLimit(v))
+        return { group, count:0, destroy: unsub }
+    })
+    const conn = ctx.state.connection as Connection | undefined
+    if (!conn) throw 'assert throttler connection'
+
+    const ts = conn[SymThrStr] = new ThrottledStream(ipGroup.group, conn[SymThrStr])
+    let closed = false
+
+    const DELAY = 1000
+    const update = _.debounce(() => {
+        const ts = conn[SymThrStr] as ThrottledStream
+        const outSpeed = roundKb(ts.getSpeed())
+        updateConnection(conn, { outSpeed, sent: ts.getBytesSent() })
+        /* in case this stream stands still for a while (before the end), we'll have neither 'sent' or 'close' events,
+        * so who will take care to updateConnection? This artificial next-call will ensure just that */
+        clearTimeout(conn[SymTimeout])
+        if (outSpeed || !closed)
+            conn[SymTimeout] = setTimeout(update, DELAY)
+    }, DELAY, { maxWait:DELAY })
+    ts.on('sent', (n: number) => {
+        totalSent += n
+        update()
+    })
+
+    ++ipGroup.count
+    ts.on('close', ()=> {
+        update.flush()
+        closed = true
+        if (--ipGroup.count) return // any left?
+        ipGroup.destroy?.()
+        delete ip2group[ctx.ip]
+    })
+
+    const bak = ctx.response.length // preserve
+    ctx.body = ctx.body.pipe(ts)
+
+    if (bak)
+        ctx.response.length = bak
+}
+
+function roundKb(n: number) {
+    return _.round(n, 1) || _.round(n, 3) // further precision if necessary
+}
+
+export let totalSent = 0
+export let totalGot = 0
+export let totalOutSpeed = 0
+export let totalInSpeed = 0
+
+let lastSent = totalSent
+let lastGot = totalGot
+let last = Date.now()
+setInterval(() => {
+    const now = Date.now()
+    const past = (now - last) / 1000 // seconds
+    last = now
+    const deltaSentKb = (totalSent - lastSent) / 1000
+    lastSent = totalSent
+    const deltaGotKb = (totalGot - lastGot) / 1000
+    lastGot = totalGot
+    totalOutSpeed = roundKb(deltaSentKb / past)
+    totalInSpeed = roundKb(deltaGotKb / past)
+}, 1000)
+
+events.on('connection', (c: Connection) =>
+    c.socket.on('data', data =>
+        totalGot += data.length ))

package/src/update.ts

@@ -0,0 +1,67 @@
+import { getRepoInfo } from './github'
+import { argv, IS_WINDOWS, VERSION } from './const'
+import { basename, dirname, join } from 'path'
+import { spawn } from 'child_process'
+import { httpsStream, onProcessExit, unzip } from './misc'
+import { renameSync, unlinkSync } from 'fs'
+import { pluginsWatcher } from './plugins'
+import { chmod, stat } from 'fs/promises'
+
+const HFS_REPO = 'rejetto/hfs'
+
+export async function getUpdate() {
+    const [latest] = await getRepoInfo(HFS_REPO + '/releases?per_page=1')
+    if (latest.name === VERSION)
+        throw "you already have the latest version: " + VERSION
+    return latest
+}
+
+export async function update() {
+    if (process.argv0.endsWith('node'))
+        throw "only binary versions are supported for now"
+    const update = await getUpdate()
+    const assetSearch = ({ win32: 'windows', darwin: 'mac', linux: 'linux' } as any)[process.platform]
+    if (!assetSearch)
+        throw "this feature doesn't support your platform: " + process.platform
+    const asset = update.assets.find((x: any) => x.name.endsWith('.zip') && x.name.includes(assetSearch))
+    if (!asset)
+        throw "asset not found"
+    const url = asset.browser_download_url
+    console.log("downloading", url)
+    const bin = process.argv[0]
+    const binPath = dirname(bin)
+    const binFile = basename(bin)
+    const newBinFile = 'new-' + binFile
+    pluginsWatcher.pause()
+    try {
+        await unzip(await httpsStream(url), path =>
+            join(binPath, path === binFile ? newBinFile : path))
+        const newBin = join(binPath, newBinFile)
+        if (!IS_WINDOWS) {
+            const { mode } = await stat(bin)
+            await chmod(newBin, mode).catch(console.error)
+        }
+        onProcessExit(() => {
+            const oldBinFile = 'old-' + binFile
+            console.log("old version is kept as", oldBinFile)
+            const oldBin = join(binPath, oldBinFile)
+            try { unlinkSync(oldBin) }
+            catch {}
+            renameSync(bin, oldBin)
+            console.log("launching new version in background", newBinFile)
+            spawn(newBin, ['--updating', binFile], { detached: true, shell: true })
+                .on('error', console.error)
+        })
+        console.log('quitting')
+        process.exit()
+    }
+    catch {
+        pluginsWatcher.unpause()
+    }
+}
+
+if (argv.updating) { // we were launched with a temporary name, restore original name to avoid breaking references
+    const bin = process.argv[0]
+    renameSync(bin, join(dirname(bin), argv.updating))
+    console.log("renamed binary file to", argv.updating)
+}

package/src/util-files.ts

@@ -0,0 +1,137 @@
+import fs from 'fs/promises'
+import { wait } from './misc'
+import { createWriteStream, mkdirSync, watch } from 'fs'
+import { basename, dirname } from 'path'
+import glob from 'fast-glob'
+import { IS_WINDOWS } from './const'
+import { execFile } from 'child_process'
+import { once, Readable } from 'stream'
+// @ts-ignore
+import unzipper from 'unzip-stream'
+
+export async function isDirectory(path: string) {
+    try { return (await fs.stat(path)).isDirectory() }
+    catch { return false }
+}
+
+export async function isFile(path: string) {
+    try { return (await fs.stat(path)).isFile() }
+    catch { return false }
+}
+
+export async function readFileBusy(path: string): Promise<string> {
+    return fs.readFile(path, 'utf8').catch(e => {
+        if ((e as any)?.code !== 'EBUSY')
+            throw e
+        console.debug('busy')
+        return wait(100).then(()=> readFileBusy(path))
+    })
+}
+
+export function watchDir(dir: string, cb: ()=>void) {
+    let watcher: ReturnType<typeof watch>
+    let paused = false
+    try {
+        watcher = watch(dir, controlledCb)
+    }
+    catch {
+        // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir
+        const base = basename(dir)
+        try {
+            watcher = watch(dirname(dir), (event,name) => {
+                if (name !== base) return
+                try {
+                    watcher.close() // if we succeed, we give up the parent watching
+                    watcher = watch(dir, controlledCb) // attempt at passing to a more specific watching
+                }
+                catch {}
+                controlledCb()
+            })
+        }
+        catch (e) {
+            console.debug(String(e))
+        }
+    }
+    return {
+        working() { return Boolean(watcher) },
+        stop() { watcher?.close() },
+        pause() { paused = true },
+        unpause() { paused = false },
+    }
+
+    function controlledCb() {
+        if (!paused)
+            cb()
+    }
+}
+
+export function dirTraversal(s?: string) {
+    return s && /(^|[/\\])\.\.($|[/\\])/.test(s)
+}
+
+export function isWindowsDrive(s?: string) {
+    return s && /^[a-zA-Z]:$/.test(s)
+}
+
+// apply this to paths that may contain \ as separator (not supported by fast-glob) and other special chars to be escaped (parenthesis)
+export function adjustStaticPathForGlob(path: string) {
+    return glob.escapePath(path.replace(/\\/g, '/'))
+}
+
+export async function* dirStream(path: string, deep?: number) {
+    if (!await isDirectory(path))
+        throw Error('ENOTDIR')
+    const dirStream = glob.stream(deep ? '**/*' : '*', {
+        cwd: path,
+        dot: true,
+        deep,
+        onlyFiles: false,
+        suppressErrors: true,
+        objectMode: true,
+    })
+    const skip = await getItemsToSkip(path)
+    for await (const entry of dirStream) {
+        let { path, dirent } = entry as any
+        if (!dirent.isDirectory() && !dirent.isFile()) continue
+        path = String(path)
+        if (!skip?.includes(path))
+            yield path
+    }
+
+    async function getItemsToSkip(path: string) {
+        if (!IS_WINDOWS) return
+        const out = await run('dir', ['/ah', '/b', path.replace(/\//g, '\\')])
+            .catch(()=>'') // error in case of no matching file
+        return out.split('\r\n').slice(0,-1)
+    }
+}
+
+export function run(cmd: string, args: string[] = []): Promise<string> {
+    return new Promise((resolve, reject) =>
+        execFile('cmd', ['/c', cmd, ...args], (err, stdout) => {
+            if (err)
+                reject(err)
+            else
+                resolve(stdout)
+        }))
+}
+
+export async function unzip(stream: Readable, cb: (path: string) => false | string) {
+    let pending: Promise<any> = Promise.resolve()
+    return new Promise(resolve =>
+        stream.pipe(unzipper.Parse())
+            .on('end', () => pending.then(resolve))
+            .on('entry', async (entry: any) => {
+                const { path, type } = entry
+                const dest = cb(path)
+                if (!dest || type !== 'File')
+                    return entry.autodrain()
+                await pending // don't overlap writings
+                console.debug('unzip', dest)
+                mkdirSync(dirname(dest), { recursive: true }) // easy way be sure to have the folder ready before proceeding
+                const thisFile = entry.pipe(createWriteStream(dest))
+                pending = once(thisFile, 'finish')
+            })
+    )
+}
+

package/src/util-generators.ts

@@ -0,0 +1,29 @@
+// callback can return undefined to skip element
+import { Readable } from 'stream'
+
+export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
+    for await (const x of generator) {
+        const res:OUT = await filterMap(x)
+        if (res !== undefined)
+            yield res as Exclude<OUT,undefined>
+    }
+}
+
+export async function asyncGeneratorToArray<T>(generator: AsyncIterable<T>): Promise<T[]> {
+    const ret: T[] = []
+    for await(const x of generator)
+        ret.push(x)
+    return ret
+}
+
+export function asyncGeneratorToReadable<T>(generator: AsyncIterable<T>) {
+    const iterator = generator[Symbol.asyncIterator]()
+    return new Readable({
+        objectMode: true,
+        read() {
+            iterator.next().then(it =>
+                this.push(it.done ? null : it.value))
+        }
+    })
+}
+

package/src/util-http.ts

@@ -0,0 +1,29 @@
+import { RequestOptions } from 'https'
+import { IncomingMessage } from 'node:http'
+import https from 'node:https'
+
+export function httpsString(url: string, options:RequestOptions={}): Promise<IncomingMessage & { ok: boolean, body: string }> {
+    return httpsStream(url, options).then(res =>
+        new Promise(resolve => {
+            let buf = ''
+            res.on('data', chunk => buf += chunk.toString())
+            res.on('end', () => resolve(Object.assign(res, {
+                ok: (res.statusCode || 400) < 400,
+                body: buf
+            })))
+        })
+    )
+}
+
+export function httpsStream(url: string, options:RequestOptions={}): Promise<IncomingMessage> {
+    return new Promise((resolve, reject) => {
+        https.request(url, options, res => {
+            if (!res.statusCode || res.statusCode >= 400)
+                throw res
+            if (res.statusCode === 302 && res.headers.location)
+                return resolve(httpsStream(res.headers.location, options))
+            resolve(res)
+        }).on('error', reject).end()
+    })
+}
+