hfs 0.26.8 → 0.26.9

package/src/serveFile.ts

@@ -1,104 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import Koa from 'koa'
-import { createReadStream, stat } from 'fs'
-import { FORBIDDEN, METHOD_NOT_ALLOWED, NO_CONTENT } from './const'
-import { getNodeName, MIME_AUTO, VfsNode } from './vfs'
-import mimetypes from 'mime-types'
-import { defineConfig } from './config'
-import { isMatch } from 'micromatch'
-import _ from 'lodash'
-import path from 'path'
-import { promisify } from 'util'
-
-const allowedReferer = defineConfig('allowed_referer', '')
-
-export function serveFileNode(node: VfsNode) : Koa.Middleware {
-    const { source, mime } = node
-    const name = getNodeName(node)
-    const mimeString = typeof mime === 'string' ? mime
-        : _.find(mime, (val,mask) => isMatch(name, mask))
-    return (ctx, next) => {
-       const allowed = allowedReferer.get()
-        if (allowed) {
-            const ref = /\/\/([^:/]+)/.exec(ctx.get('referer'))?.[1] // extract host from url
-            if (ref && ref !== host() // automatic accept if referer is basically the hosting domain
-            && !isMatch(ref, allowed))
-                return ctx.status = FORBIDDEN
-
-            function host() {
-                const s = ctx.get('host')
-                return s[0] === '[' ? s.slice(1, s.indexOf(']')) : s?.split(':')[0]
-            }
-        }
-
-        ctx.vfsNode = node // useful to tell service files from files shared by the user
-        return serveFile(source||'', mimeString)(ctx, next)
-    }
-}
-
-const mimeCfg = defineConfig<Record<string,string>>('mime', { '*': 'auto' })
-
-export function serveFile(source:string, mime?:string, content?: string | Buffer) : Koa.Middleware {
-    return async (ctx) => {
-        if (!source)
-            return
-        const fn = path.basename(source)
-        mime = mime ?? _.find(mimeCfg.get(), (v,k) => k>'' && isMatch(fn, k)) // isMatch throws on an empty string
-        if (mime === MIME_AUTO)
-            mime = mimetypes.lookup(source) || ''
-        if (mime)
-            ctx.type = mime
-        if (ctx.method === 'OPTIONS') {
-            ctx.status = NO_CONTENT
-            ctx.set({ Allow: 'OPTIONS, GET, HEAD' })
-            return
-        }
-        if (ctx.method !== 'GET')
-            return ctx.status = METHOD_NOT_ALLOWED
-        try {
-            const stats = await promisify(stat)(source) // using fs's function instead of fs/promises, because only the former is supported by pkg
-            ctx.set('Last-Modified', stats.mtime.toUTCString())
-            ctx.fileSource = source
-            ctx.status = 200
-            if (ctx.fresh)
-                return ctx.status = 304
-            if (content !== undefined)
-                return ctx.body = content
-            const range = getRange(ctx, stats.size)
-            ctx.body = createReadStream(source, range)
-        }
-        catch {
-            return ctx.status = 404
-        }
-    }
-}
-
-export function getRange(ctx: Koa.Context, totalSize: number) {
-    ctx.set('Accept-Ranges', 'bytes')
-    const { range } = ctx.request.header
-    if (!range) {
-        ctx.response.length = totalSize
-        return
-    }
-    const ranges = range.split('=')[1]
-    if (ranges.includes(','))
-        return ctx.throw(400, 'multi-range not supported')
-    let bytes = ranges?.split('-')
-    if (!bytes?.length)
-        return ctx.throw(400, 'bad range')
-    const max = totalSize - 1
-    const start = bytes[0] ? Number(bytes[0]) : Math.max(0, totalSize-Number(bytes[1])) // a negative start is relative to the end
-    const end = bytes[0] ? Number(bytes[1] || max) : max
-    // we don't support last-bytes without knowing max
-    if (isNaN(end) && isNaN(max) || end > max || start > max) {
-        ctx.status = 416
-        ctx.set('Content-Range', `bytes ${totalSize}`)
-        ctx.body = 'Requested Range Not Satisfiable'
-        return
-    }
-    ctx.status = 206
-    ctx.set('Content-Range', `bytes ${start}-${isNaN(end) ? '' : end}/${isNaN(totalSize) ? '*' : totalSize}`)
-    ctx.response.length = end - start + 1
-    return { start, end }
-}

package/src/serveGuiFiles.ts

@@ -1,95 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import Koa from 'koa'
-import fs from 'fs/promises'
-import { METHOD_NOT_ALLOWED, NO_CONTENT, PLUGINS_PUB_URI } from './const'
-import { serveFile } from './serveFile'
-import { mapPlugins } from './plugins'
-import { refresh_session } from './api.auth'
-import { ApiError } from './apiMiddleware'
-import { join, extname } from 'path'
-import { getOrSet } from './misc'
-
-// in case of dev env we have our static files within the 'dist' folder'
-const DEV_STATIC = process.env.DEV ? 'dist/' : ''
-
-function serveStatic(uri: string): Koa.Middleware {
-    const folder = uri.slice(2,-1) // we know folder is very similar to uri
-    const cache: Record<string, Promise<string>> = {}
-    return async (ctx, next) => {
-        if(ctx.method === 'OPTIONS') {
-            ctx.status = NO_CONTENT
-            ctx.set({ Allow: 'OPTIONS, GET' })
-            return
-        }
-        if (ctx.method !== 'GET')
-            return ctx.status = METHOD_NOT_ALLOWED
-        const serveApp = shouldServeApp(ctx)
-        const fullPath = join(__dirname, '..', DEV_STATIC, folder, serveApp ? '/index.html': ctx.path)
-        const content = await getOrSet(cache, ctx.path, async () => {
-            const data = await fs.readFile(fullPath).catch(() => null)
-            return serveApp || !data ? data
-                : adjustBundlerLinks(ctx.path, uri, data)
-        })
-        if (content === null)
-            return ctx.status = 404
-        if (!serveApp)
-            return serveFile(fullPath, 'auto', content)(ctx, next)
-        // we don't cache the index as it's small and may prevent plugins change to apply
-        ctx.body = await treatIndex(ctx, String(content), uri)
-        ctx.type = 'html'
-        ctx.set('Cache-Control', 'no-store, no-cache, must-revalidate')
-    }
-}
-
-function shouldServeApp(ctx: Koa.Context) {
-    return ctx.state.serveApp ||= ctx.path.endsWith('/')
-}
-
-function adjustBundlerLinks(path: string, uri: string, data: string | Buffer) {
-    const ext = extname(path)
-    return ext && !ext.match(/\.(css|html|js|ts|scss)/) ? data
-        : String(data).replace(/((?:import | from )['"])\//g, `$1${uri}`)
-}
-
-async function treatIndex(ctx: Koa.Context, body: string, filesUri: string) {
-    const session = await refresh_session({}, ctx)
-    ctx.set('etag', '')
-    return body
-        .replace(/((?:src|href) *= *['"])\/?(?![a-z]+:\/\/)/g, '$1' + filesUri)
-        .replace('_HFS_SESSION_', session instanceof ApiError ? 'null' : JSON.stringify(session))
-        // replacing this text allow us to avoid injecting in frontends that don't support plugins. Don't use a <--comment--> or it will be removed by webpack
-        .replace('_HFS_PLUGINS_', pluginsInjection)
-}
-
-function serveProxied(port: string | undefined, uri: string) { // used for development only
-    if (!port)
-        return
-    console.debug('proxied on port', port)
-    let proxy: Koa.Middleware
-    import('koa-better-http-proxy').then(lib => // dynamic import to avoid having this in final distribution
-        proxy = lib.default('127.0.0.1:'+port, {
-            proxyReqPathResolver: (ctx) =>
-                shouldServeApp(ctx) ? '/' : ctx.path,
-            userResDecorator(res, data, ctx) {
-                return shouldServeApp(ctx) ? treatIndex(ctx, String(data), uri)
-                    : adjustBundlerLinks(ctx.path, uri, data)
-            }
-        }) )
-    return function() { //@ts-ignore
-        return proxy.apply(this,arguments)
-    }
-}
-
-function pluginsInjection() {
-    const css = mapPlugins((plug,k) =>
-        plug.frontend_css?.map(f => PLUGINS_PUB_URI + k + '/' + f)).flat().filter(Boolean)
-    const js = mapPlugins((plug,k) =>
-        plug.frontend_js?.map(f => PLUGINS_PUB_URI + k + '/' + f)).flat().filter(Boolean)
-    return css.map(uri => `\n<link rel='stylesheet' type='text/css' href='${uri}'/>`).join('')
-        + js.map(uri => `\n<script defer src='${uri}'></script>`).join('')
-}
-
-export function serveGuiFiles(proxyPort:string | undefined, uri:string) {
-    return serveProxied(proxyPort, uri) || serveStatic(uri)
-}

package/src/sse.ts

@@ -1,29 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import Koa from 'koa'
-import { Transform } from 'stream'
-
-export default function createSSE(ctx: Koa.Context) {
-    const { socket } = ctx.req
-    socket.setTimeout(0)
-    socket.setNoDelay(true)
-    socket.setKeepAlive(true)
-    ctx.set({
-        'Content-Type': 'text/event-stream',
-        'Cache-Control': 'no-cache',
-        'Connection': 'keep-alive',
-        'X-Accel-Buffering': 'no', // avoid buffering when reverse-proxied through nginx
-    })
-    ctx.status = 200
-    return ctx.body = new Transform({
-        objectMode: true,
-        transform(chunk, encoding, cb) {
-            this.push(`data: ${JSON.stringify(chunk)}\n\n`)
-            cb()
-        },
-        flush(cb) {
-            this.push('data:\n\n')
-            cb()
-        }
-    })
-}

package/src/throttler.ts

@@ -1,106 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import { Readable } from 'stream'
-import Koa from 'koa'
-import { ThrottledStream, ThrottleGroup } from './ThrottledStream'
-import { defineConfig } from './config'
-import { getOrSet, isLocalHost } from './misc'
-import { Connection, updateConnection } from './connections'
-import _ from 'lodash'
-import events from './events'
-
-const mainThrottleGroup = new ThrottleGroup(Infinity)
-
-defineConfig('max_kbps', Infinity).sub(v =>
-    mainThrottleGroup.updateLimit(v))
-
-const ip2group: Record<string, {
-    count: number
-    group: ThrottleGroup
-    destroy: () => void
-}> = {}
-
-const SymThrStr = Symbol('stream')
-const SymTimeout = Symbol('timeout')
-
-const maxKbpsPerIp = defineConfig('max_kbps_per_ip', Infinity)
-
-export const throttler: Koa.Middleware = async (ctx, next) => {
-    await next()
-    const { body } = ctx
-    if (!body || !(body instanceof Readable))
-        return
-    // we wrap the stream also for unlimited connections to get speed and other features
-    const ipGroup = getOrSet(ip2group, ctx.ip, ()=> {
-        const doLimit = ctx.state.account?.ignore_limits || isLocalHost(ctx) ? undefined : true
-        const group = new ThrottleGroup(Infinity, doLimit && mainThrottleGroup)
-
-        const unsub = doLimit && maxKbpsPerIp.sub(v =>
-            group.updateLimit(v))
-        return { group, count:0, destroy: unsub }
-    })
-    const conn = ctx.state.connection as Connection | undefined
-    if (!conn) throw 'assert throttler connection'
-
-    const ts = conn[SymThrStr] = new ThrottledStream(ipGroup.group, conn[SymThrStr])
-    let closed = false
-
-    const DELAY = 1000
-    const update = _.debounce(() => {
-        const ts = conn[SymThrStr] as ThrottledStream
-        const outSpeed = roundKb(ts.getSpeed())
-        updateConnection(conn, { outSpeed, sent: ts.getBytesSent() })
-        /* in case this stream stands still for a while (before the end), we'll have neither 'sent' or 'close' events,
-        * so who will take care to updateConnection? This artificial next-call will ensure just that */
-        clearTimeout(conn[SymTimeout])
-        if (outSpeed || !closed)
-            conn[SymTimeout] = setTimeout(update, DELAY)
-    }, DELAY, { maxWait:DELAY })
-    ts.on('sent', (n: number) => {
-        totalSent += n
-        update()
-    })
-
-    ++ipGroup.count
-    ts.on('close', ()=> {
-        update.flush()
-        closed = true
-        if (--ipGroup.count) return // any left?
-        ipGroup.destroy?.()
-        delete ip2group[ctx.ip]
-    })
-
-    const bak = ctx.response.length // preserve
-    ctx.body = ctx.body.pipe(ts)
-
-    if (bak)
-        ctx.response.length = bak
-}
-
-function roundKb(n: number) {
-    return _.round(n, 1) || _.round(n, 3) // further precision if necessary
-}
-
-export let totalSent = 0
-export let totalGot = 0
-export let totalOutSpeed = 0
-export let totalInSpeed = 0
-
-let lastSent = totalSent
-let lastGot = totalGot
-let last = Date.now()
-setInterval(() => {
-    const now = Date.now()
-    const past = (now - last) / 1000 // seconds
-    last = now
-    const deltaSentKb = (totalSent - lastSent) / 1000
-    lastSent = totalSent
-    const deltaGotKb = (totalGot - lastGot) / 1000
-    lastGot = totalGot
-    totalOutSpeed = roundKb(deltaSentKb / past)
-    totalInSpeed = roundKb(deltaGotKb / past)
-}, 1000)
-
-events.on('connection', (c: Connection) =>
-    c.socket.on('data', data =>
-        totalGot += data.length ))

package/src/update.ts

@@ -1,67 +0,0 @@
-import { getRepoInfo } from './github'
-import { argv, IS_WINDOWS, VERSION } from './const'
-import { basename, dirname, join } from 'path'
-import { spawn } from 'child_process'
-import { httpsStream, onProcessExit, unzip } from './misc'
-import { renameSync, unlinkSync } from 'fs'
-import { pluginsWatcher } from './plugins'
-import { chmod, stat } from 'fs/promises'
-
-const HFS_REPO = 'rejetto/hfs'
-
-export async function getUpdate() {
-    const [latest] = await getRepoInfo(HFS_REPO + '/releases?per_page=1')
-    if (latest.name === VERSION)
-        throw "you already have the latest version: " + VERSION
-    return latest
-}
-
-export async function update() {
-    if (process.argv0.endsWith('node'))
-        throw "only binary versions are supported for now"
-    const update = await getUpdate()
-    const assetSearch = ({ win32: 'windows', darwin: 'mac', linux: 'linux' } as any)[process.platform]
-    if (!assetSearch)
-        throw "this feature doesn't support your platform: " + process.platform
-    const asset = update.assets.find((x: any) => x.name.endsWith('.zip') && x.name.includes(assetSearch))
-    if (!asset)
-        throw "asset not found"
-    const url = asset.browser_download_url
-    console.log("downloading", url)
-    const bin = process.argv[0]
-    const binPath = dirname(bin)
-    const binFile = basename(bin)
-    const newBinFile = 'new-' + binFile
-    pluginsWatcher.pause()
-    try {
-        await unzip(await httpsStream(url), path =>
-            join(binPath, path === binFile ? newBinFile : path))
-        const newBin = join(binPath, newBinFile)
-        if (!IS_WINDOWS) {
-            const { mode } = await stat(bin)
-            await chmod(newBin, mode).catch(console.error)
-        }
-        onProcessExit(() => {
-            const oldBinFile = 'old-' + binFile
-            console.log("old version is kept as", oldBinFile)
-            const oldBin = join(binPath, oldBinFile)
-            try { unlinkSync(oldBin) }
-            catch {}
-            renameSync(bin, oldBin)
-            console.log("launching new version in background", newBinFile)
-            spawn(newBin, ['--updating', binFile], { detached: true, shell: true })
-                .on('error', console.error)
-        })
-        console.log('quitting')
-        process.exit()
-    }
-    catch {
-        pluginsWatcher.unpause()
-    }
-}
-
-if (argv.updating) { // we were launched with a temporary name, restore original name to avoid breaking references
-    const bin = process.argv[0]
-    renameSync(bin, join(dirname(bin), argv.updating))
-    console.log("renamed binary file to", argv.updating)
-}

package/src/util-files.ts

@@ -1,137 +0,0 @@
-import fs from 'fs/promises'
-import { wait } from './misc'
-import { createWriteStream, mkdirSync, watch } from 'fs'
-import { basename, dirname } from 'path'
-import glob from 'fast-glob'
-import { IS_WINDOWS } from './const'
-import { execFile } from 'child_process'
-import { once, Readable } from 'stream'
-// @ts-ignore
-import unzipper from 'unzip-stream'
-
-export async function isDirectory(path: string) {
-    try { return (await fs.stat(path)).isDirectory() }
-    catch { return false }
-}
-
-export async function isFile(path: string) {
-    try { return (await fs.stat(path)).isFile() }
-    catch { return false }
-}
-
-export async function readFileBusy(path: string): Promise<string> {
-    return fs.readFile(path, 'utf8').catch(e => {
-        if ((e as any)?.code !== 'EBUSY')
-            throw e
-        console.debug('busy')
-        return wait(100).then(()=> readFileBusy(path))
-    })
-}
-
-export function watchDir(dir: string, cb: ()=>void) {
-    let watcher: ReturnType<typeof watch>
-    let paused = false
-    try {
-        watcher = watch(dir, controlledCb)
-    }
-    catch {
-        // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir
-        const base = basename(dir)
-        try {
-            watcher = watch(dirname(dir), (event,name) => {
-                if (name !== base) return
-                try {
-                    watcher.close() // if we succeed, we give up the parent watching
-                    watcher = watch(dir, controlledCb) // attempt at passing to a more specific watching
-                }
-                catch {}
-                controlledCb()
-            })
-        }
-        catch (e) {
-            console.debug(String(e))
-        }
-    }
-    return {
-        working() { return Boolean(watcher) },
-        stop() { watcher?.close() },
-        pause() { paused = true },
-        unpause() { paused = false },
-    }
-
-    function controlledCb() {
-        if (!paused)
-            cb()
-    }
-}
-
-export function dirTraversal(s?: string) {
-    return s && /(^|[/\\])\.\.($|[/\\])/.test(s)
-}
-
-export function isWindowsDrive(s?: string) {
-    return s && /^[a-zA-Z]:$/.test(s)
-}
-
-// apply this to paths that may contain \ as separator (not supported by fast-glob) and other special chars to be escaped (parenthesis)
-export function adjustStaticPathForGlob(path: string) {
-    return glob.escapePath(path.replace(/\\/g, '/'))
-}
-
-export async function* dirStream(path: string, deep?: number) {
-    if (!await isDirectory(path))
-        throw Error('ENOTDIR')
-    const dirStream = glob.stream(deep ? '**/*' : '*', {
-        cwd: path,
-        dot: true,
-        deep,
-        onlyFiles: false,
-        suppressErrors: true,
-        objectMode: true,
-    })
-    const skip = await getItemsToSkip(path)
-    for await (const entry of dirStream) {
-        let { path, dirent } = entry as any
-        if (!dirent.isDirectory() && !dirent.isFile()) continue
-        path = String(path)
-        if (!skip?.includes(path))
-            yield path
-    }
-
-    async function getItemsToSkip(path: string) {
-        if (!IS_WINDOWS) return
-        const out = await run('dir', ['/ah', '/b', path.replace(/\//g, '\\')])
-            .catch(()=>'') // error in case of no matching file
-        return out.split('\r\n').slice(0,-1)
-    }
-}
-
-export function run(cmd: string, args: string[] = []): Promise<string> {
-    return new Promise((resolve, reject) =>
-        execFile('cmd', ['/c', cmd, ...args], (err, stdout) => {
-            if (err)
-                reject(err)
-            else
-                resolve(stdout)
-        }))
-}
-
-export async function unzip(stream: Readable, cb: (path: string) => false | string) {
-    let pending: Promise<any> = Promise.resolve()
-    return new Promise(resolve =>
-        stream.pipe(unzipper.Parse())
-            .on('end', () => pending.then(resolve))
-            .on('entry', async (entry: any) => {
-                const { path, type } = entry
-                const dest = cb(path)
-                if (!dest || type !== 'File')
-                    return entry.autodrain()
-                await pending // don't overlap writings
-                console.debug('unzip', dest)
-                mkdirSync(dirname(dest), { recursive: true }) // easy way be sure to have the folder ready before proceeding
-                const thisFile = entry.pipe(createWriteStream(dest))
-                pending = once(thisFile, 'finish')
-            })
-    )
-}
-

package/src/util-generators.ts

@@ -1,29 +0,0 @@
-// callback can return undefined to skip element
-import { Readable } from 'stream'
-
-export async function* filterMapGenerator<IN,OUT>(generator: AsyncIterableIterator<IN>, filterMap: (el: IN) => Promise<OUT>) {
-    for await (const x of generator) {
-        const res:OUT = await filterMap(x)
-        if (res !== undefined)
-            yield res as Exclude<OUT,undefined>
-    }
-}
-
-export async function asyncGeneratorToArray<T>(generator: AsyncIterable<T>): Promise<T[]> {
-    const ret: T[] = []
-    for await(const x of generator)
-        ret.push(x)
-    return ret
-}
-
-export function asyncGeneratorToReadable<T>(generator: AsyncIterable<T>) {
-    const iterator = generator[Symbol.asyncIterator]()
-    return new Readable({
-        objectMode: true,
-        read() {
-            iterator.next().then(it =>
-                this.push(it.done ? null : it.value))
-        }
-    })
-}
-

package/src/util-http.ts

@@ -1,29 +0,0 @@
-import { RequestOptions } from 'https'
-import { IncomingMessage } from 'node:http'
-import https from 'node:https'
-
-export function httpsString(url: string, options:RequestOptions={}): Promise<IncomingMessage & { ok: boolean, body: string }> {
-    return httpsStream(url, options).then(res =>
-        new Promise(resolve => {
-            let buf = ''
-            res.on('data', chunk => buf += chunk.toString())
-            res.on('end', () => resolve(Object.assign(res, {
-                ok: (res.statusCode || 400) < 400,
-                body: buf
-            })))
-        })
-    )
-}
-
-export function httpsStream(url: string, options:RequestOptions={}): Promise<IncomingMessage> {
-    return new Promise((resolve, reject) => {
-        https.request(url, options, res => {
-            if (!res.statusCode || res.statusCode >= 400)
-                throw res
-            if (res.statusCode === 302 && res.headers.location)
-                return resolve(httpsStream(res.headers.location, options))
-            resolve(res)
-        }).on('error', reject).end()
-    })
-}
-