hfs 0.26.8 → 0.27.2

package/src/vfs.ts

@@ -1,258 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import fs from 'fs/promises'
-import { basename, join } from 'path'
-import { isMatch } from 'micromatch'
-import { dirStream, dirTraversal, enforceFinal, getOrSet, isDirectory, typedKeys } from './misc'
-import Koa from 'koa'
-import _ from 'lodash'
-import { defineConfig, setConfig } from './config'
-import { FORBIDDEN, IS_WINDOWS, UNAUTHORIZED } from './const'
-import events from './events'
-import { getCurrentUsernameExpanded } from './perm'
-import { with_ } from './misc'
-
-const WHO_ANYONE = true
-const WHO_NO_ONE = false
-const WHO_ANY_ACCOUNT = '*'
-type AccountList = string[]
-type Who = typeof WHO_ANYONE
-    | typeof WHO_NO_ONE
-    | typeof WHO_ANY_ACCOUNT
-    | AccountList
-
-interface VfsPerm {
-    can_see: Who
-    can_read: Who
-}
-
-type Masks = Record<string, VfsNode>
-
-export interface VfsNode extends Partial<VfsPerm> {
-    name?: string
-    source?: string
-    children?: VfsNode[]
-    default?: string
-    mime?: string | Record<string,string>
-    rename?: Record<string, string>
-    masks?: Masks // express fields for descendants that are not in the tree
-    // fields that are only filled at run-time
-    isTemp?: true // this node doesn't belong to the tree and was created by necessity
-    url?: string // what url brought to this node
-    parents?: VfsNode[]
-    original?: VfsNode // if this is a temp node but reflecting an existing node
-}
-
-export const defaultPerms: VfsPerm = {
-    can_see: WHO_ANYONE,
-    can_read: WHO_ANYONE,
-}
-
-export const MIME_AUTO = 'auto'
-
-function inheritFromParent(parent: VfsNode, child: VfsNode) {
-    for (const k of typedKeys(defaultPerms)) {
-        const v = parent[k]
-        if (v !== undefined)
-            child[k] ??= v
-    }
-    if (typeof parent.mime === 'object' && typeof child.mime === 'object')
-        _.defaults(child.mime, parent.mime)
-    else
-        child.mime ||= parent.mime
-    return child
-}
-
-export async function urlToNode(url: string, ctx?: Koa.Context, parent: VfsNode=vfs) : Promise<VfsNode | undefined> {
-    let initialSlashes = 0
-    while (url[initialSlashes] === '/')
-        initialSlashes++
-    let nextSlash = url.indexOf('/', initialSlashes)
-    const name = decodeURIComponent(url.slice(initialSlashes, nextSlash < 0 ? undefined : nextSlash))
-    if (!name)
-        return parent
-    const rest = nextSlash < 0 ? '' : url.slice(nextSlash+1, url.endsWith('/') ? -1 : undefined)
-    if (dirTraversal(name) || /[\/]/.test(name)) {
-        if (ctx)
-            ctx.status = 418
-        return
-    }
-    const parents = parent.parents || [] // don't waste time cloning the array, as we won't keep intermediate nodes
-    const ret: VfsNode = {
-        isTemp: true,
-        url: enforceFinal('/', parent.url || '') + name,
-        parents,
-    }
-    parents.push(parent)
-    inheritFromParent(parent, ret)
-    inheritMasks(ret, parent, name)
-    applyMasks(ret, parent, name)
-    // does the tree node have a child that goes by this name?
-    const sameName = !IS_WINDOWS ? (x:string) => x === name // easy
-        : with_(name.toLowerCase(), lc =>
-            (x: string) => x.toLowerCase() === lc)
-    const child = parent.children?.find(x => sameName(getNodeName(x)))
-    if (child)  // yes
-        return urlToNode(rest, ctx, Object.assign(ret, child, { original: child }))
-    // not in the tree, we can see consider continuing on the disk
-    if (!parent.source) return // but then we need the current node to be linked to the disk, otherwise, we give up
-    let onDisk = name
-    if (parent.rename) { // reverse the mapping
-        for (const [from, to] of Object.entries(parent.rename))
-            if (name === to) {
-                onDisk = from
-                break // found, search no more
-            }
-        ret.rename = renameUnderPath(parent.rename, name)
-    }
-    ret.source = enforceFinal('/', parent.source) + onDisk
-    if (parent.default)
-        inheritFromParent({ mime: { '*': MIME_AUTO } }, ret)
-    if (rest)
-        return urlToNode(rest, ctx, ret)
-    if (ret.source)
-        try { await fs.stat(ret.source) } // check existence
-        catch { return }
-    return ret
-}
-
-export let vfs: VfsNode = {}
-defineConfig<VfsNode>('vfs', {}).sub(data =>
-    vfs = data)
-
-export function saveVfs() {
-    return setConfig({ vfs: _.cloneDeep(vfs) }, true)
-}
-
-export function getNodeName(node: VfsNode) {
-    return node.name
-        || node.source && (
-            /^[a-zA-Z]:\\?$/.test(node.source) && node.source.slice(0, 2)
-            || basename(node.source)
-            || node.source
-        )
-        || '' // should happen only for root
-}
-
-export async function nodeIsDirectory(node: VfsNode) {
-    return Boolean(!node.source || await isDirectory(node.source))
-}
-
-export function hasPermission(node: VfsNode, perm: keyof VfsPerm, ctx: Koa.Context): boolean {
-    return matchWho(node[perm] ?? defaultPerms[perm], ctx)
-        && (perm !== 'can_see' || hasPermission(node, 'can_read', ctx)) // for can_see you must also can_read
-}
-
-export async function* walkNode(parent:VfsNode, ctx?: Koa.Context, depth:number=0, prefixPath:string=''): AsyncIterableIterator<VfsNode> {
-    const { children, source } = parent
-    if (children)
-        for (let idx = 0; idx < children.length; idx++) {
-            const child = children[idx]
-            yield* workItem({
-                ...child,
-                name: prefixPath ? (prefixPath + getNodeName(child)) : child.name
-            }, depth > 0 && await nodeIsDirectory(child).catch(() => false))
-        }
-    if (!source)
-        return
-    try {
-        for await (const path of dirStream(source, depth)) {
-            if (ctx?.req.aborted)
-                return
-            const renamed = parent.rename?.[path]
-            yield* workItem({
-                name: prefixPath + (renamed || path),
-                source: join(source, path),
-                rename: renameUnderPath(parent.rename, path),
-            })
-        }
-    }
-    catch(e) {
-        console.debug('glob', source, e) // ENOTDIR, or lacking permissions
-    }
-
-    // item will be changed, so be sure to pass a temp node
-    async function* workItem(item: VfsNode, recur=false) {
-        const name = getNodeName(item)
-        // we basename for depth>0 where we already have the rest of the path in the parent's url, and would be duplicated
-        const virtualBasename = basename(name)
-        const url = enforceFinal('/', parent.url || '') + virtualBasename
-        Object.assign(item, {
-            isTemp: true,
-            url,
-            parents: [ ...parent.parents||[], parent],
-        })
-        inheritFromParent(parent, item)
-        applyMasks(item, parent, virtualBasename)
-        if (ctx && !hasPermission(item, 'can_see', ctx))
-            return
-        yield item
-        if (!recur) return
-        inheritMasks(item, parent, virtualBasename)
-        yield* walkNode(item, ctx, depth - 1, name + '/')
-    }
-}
-function applyMasks(item: VfsNode, parent: VfsNode, virtualBasename: string) {
-    const { masks } = parent
-    if (!masks) return
-    for (const k in masks)
-        if (k.startsWith('**/') && isMatch(virtualBasename, k.slice(3))
-        || !k.includes('/') && isMatch(virtualBasename, k))
-            Object.assign(item, masks[k])
-}
-
-function inheritMasks(item: VfsNode, parent: VfsNode, virtualBasename:string) {
-    const { masks } = parent
-    if (!masks) return
-    const o: Masks = {}
-    for (const k in masks)
-        if (k.startsWith('**/'))
-            o[k.slice(3)] = masks[k]
-        else if (k.startsWith(virtualBasename+'/'))
-            o[k.slice(virtualBasename.length+1)] = masks[k]
-    if (Object.keys(o).length)
-        item.masks = o
-}
-
-function renameUnderPath(rename:undefined | Record<string,string>, path: string) {
-    if (!rename) return rename
-    const match = path+'/'
-    rename = Object.fromEntries(Object.entries(rename).map(([k, v]) =>
-        [k.startsWith(match) ? k.slice(match.length) : '', v]))
-    delete rename['']
-    return _.isEmpty(rename) ? undefined : rename
-}
-
-function matchWho(who: Who, ctx: Koa.Context) {
-    return who === WHO_ANYONE
-        || who === WHO_ANY_ACCOUNT && Boolean(ctx.state.account)
-        || Array.isArray(who) && (() => // check if I or any ancestor match `who`, but cache ancestors' usernames inside context state
-            getOrSet(ctx.state, 'usernames', () => getCurrentUsernameExpanded(ctx)).some((u:string) =>
-                who.includes(u) ))()
-}
-
-export function cantReadStatusCode(node: VfsNode) {
-    return node.can_read === false ? FORBIDDEN : UNAUTHORIZED
-}
-
-events.on('accountRenamed', (from, to) => {
-    recur(vfs)
-    saveVfs()
-
-    function recur(n: VfsNode) {
-        replace(n.can_see)
-        replace(n.can_read)
-
-        if (n.masks)
-            Object.values(n.masks).forEach(recur)
-        n.children?.forEach(recur)
-    }
-
-    function replace(a?: Who) {
-        if (!Array.isArray(a)) return
-        for (let i=0; i < a.length; i++)
-            if (a[i] === from)
-                a[i] = to
-    }
-
-})

package/src/watchLoad.ts

@@ -1,75 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import { FSWatcher, watch } from 'fs'
-import fs from 'fs/promises'
-import yaml from 'yaml'
-import { debounceAsync, readFileBusy } from './misc'
-
-export type WatchLoadCanceller = () => void
-
-interface Options { failedOnFirstAttempt?: ()=>void }
-
-type WriteFile = typeof fs.writeFile
-interface WatchLoadReturn { unwatch:WatchLoadCanceller, save:WriteFile }
-export function watchLoad(path:string, parser:(data:any)=>void|Promise<void>, { failedOnFirstAttempt }:Options={}): WatchLoadReturn {
-    let doing = false
-    let watcher: FSWatcher | undefined
-    const debounced = debounceAsync(load, 500, { leading: true })
-    let retry: NodeJS.Timeout
-    let saving: Promise<unknown> | undefined
-    let lastStats: any
-    init().then(ok => ok || failedOnFirstAttempt?.())
-    return {
-        unwatch(){
-            watcher?.close()
-            clearTimeout(retry)
-            watcher = undefined
-        },
-        save(...args:Parameters<WriteFile>) {
-            return Promise.resolve(saving).then(() => // wait in case another is ongoing
-                saving = fs.writeFile(...args).finally(() => // save but also keep track of the current operation
-                    saving = undefined)) // clear
-        }
-    }
-
-    async function init() {
-        try {
-            debounced().then()
-            watcher = watch(path, ()=> {
-                if (!saving)
-                    debounced().then()
-            })
-            return true // used actually just by the first invocation
-        }
-        catch(e) {
-            retry = setTimeout(init, 3_000) // manual watching until watch is successful
-        }
-    }
-
-    async function load(){
-        if (doing) return
-        doing = true
-        let data: any
-        try {
-            try { // I've seen watch() firing 'change' without any change, so we'll check if any change is detectable before going on
-                const stats = await fs.stat(path)
-                if (stats.mtimeMs === lastStats?.mtimeMs) return
-                lastStats = stats
-
-                data = await readFileBusy(path)
-                console.debug('loaded', path)
-            }
-            catch (e: any) {
-                if (e.code === 'EPERM')
-                    console.error("missing permissions on file", path) // warn user, who could be clueless about this problem
-                return // ignore read errors
-            }
-            if (path.endsWith('.yaml'))
-                data = yaml.parse(data)
-            await parser(data)
-        }
-        finally {
-            doing = false
-        }
-    }
-}

package/src/zip.ts

@@ -1,69 +0,0 @@
-// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
-
-import { getNodeName, hasPermission, nodeIsDirectory, urlToNode, VfsNode, walkNode } from './vfs'
-import Koa from 'koa'
-import { filterMapGenerator, pattern2filter } from './misc'
-import { QuickZipStream } from './QuickZipStream'
-import { createReadStream } from 'fs'
-import fs from 'fs/promises'
-import { defineConfig } from './config'
-import { dirname } from 'path'
-import { getRange } from './serveFile'
-
-export async function zipStreamFromFolder(node: VfsNode, ctx: Koa.Context) {
-    ctx.status = 200
-    ctx.mime = 'zip'
-    const name = getNodeName(node)
-    ctx.attachment((name || 'archive') + '.zip')
-    const filter = pattern2filter(String(ctx.query.search||''))
-    const { list } = ctx.query
-    const walker = !list ? walkNode(node, ctx, Infinity)
-        : (async function*(): AsyncIterableIterator<VfsNode> {
-            for await (const el of String(list).split('*')) { // we are using * as separator because it cannot be used in a file name and doesn't need url encoding
-                const subNode = await urlToNode(el, ctx, node)
-                if (!subNode || !hasPermission(subNode,'can_read',ctx))
-                    continue
-                if (await nodeIsDirectory(subNode)) {// a directory needs to walked
-                    yield* walkNode(subNode, ctx, Infinity, el + '/')
-                    continue
-                }
-                let folder = dirname(el)
-                folder = folder === '.' ? '' : folder + '/'
-                yield { ...subNode, name: folder + getNodeName(subNode) } // reflect relative path in archive, otherwise way may have name-clashes
-            }
-        })()
-    const mappedWalker = filterMapGenerator(walker, async (el:VfsNode) => {
-        const { source } = el
-        const name = getNodeName(el)
-        if (!source || ctx.req.aborted || !filter(name))
-            return
-        try {
-            const st = await fs.stat(source)
-            if (!st || !st.isFile())
-                return
-            return {
-                path: name,
-                size: st.size,
-                ts: st.mtime || st.ctime,
-                mode: st.mode,
-                sourcePath: source,
-                getData: () => createReadStream(source, { start: 0 , end: Math.max(0, st.size-1) })
-            }
-        }
-        catch {}
-    })
-    const zip = new QuickZipStream(mappedWalker)
-    const time = 1000 * zipSeconds.get()
-    const size = await zip.calculateSize(time)
-    ctx.response.length = size
-    const range = getRange(ctx, size) // keep var size as ctx.response.length won't preserve a NaN
-    if (ctx.status >= 400)
-        return
-    if (range)
-        zip.applyRange(range.start, range.end)
-    ctx.body = zip
-    ctx.req.on('close', ()=> zip.destroy())
-    ctx.state.archive = 'zip'
-}
-
-const zipSeconds = defineConfig('zip_calculate_size_for_seconds', 1)