hfs 0.26.7 → 0.26.8

package/frontend/src/state.ts

@@ -0,0 +1,82 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import _ from 'lodash'
+import { proxy, useSnapshot } from 'valtio'
+import { subscribeKey } from 'valtio/utils'
+import { apiCall } from './api'
+import { DirList } from './BrowseFiles'
+
+export const state = proxy<{
+    stopSearch?: ()=>void,
+    stoppedSearch?: boolean,
+    iconsClass: string,
+    username: string,
+    list: DirList,
+    filteredList?: DirList,
+    loading: boolean,
+    error?: string,
+    listReloader: number,
+    patternFilter: string,
+    showFilter: boolean,
+    selected: Record<string,true>, // optimization: by using an object instead of an array, components are not rendered when the array changes, but only when their specific property change
+    remoteSearch: string,
+    sortBy: string,
+    invertOrder: boolean,
+    foldersFirst: boolean,
+    theme: string,
+    adminUrl?: string,
+    serverConfig?: any,
+    loginRequired?: boolean, // force user to login before proceeding
+    messageOnly?: string, // no gui, just show this message
+}>({
+    iconsClass: '',
+    username: '',
+    list: [],
+    filteredList: undefined,
+    loading: false,
+    listReloader: 0,
+    patternFilter: '',
+    showFilter: false,
+    selected: {},
+    remoteSearch: '',
+    sortBy: 'name',
+    invertOrder: false,
+    foldersFirst: true,
+    theme: '',
+})
+
+export function useSnapState() {
+    return useSnapshot(state)
+}
+
+const SETTINGS_KEY = 'hfs_settings'
+const SETTINGS_TO_STORE: (keyof typeof state)[] = ['sortBy', 'invertOrder', 'foldersFirst', 'theme']
+
+loadSettings()
+for (const k of SETTINGS_TO_STORE)
+    subscribeKey(state, k, storeSettings)
+
+// load server config
+setTimeout(() =>
+    apiCall('config', {}, { noModal: true }).then(res =>
+        state.serverConfig = res) )
+
+function loadSettings() {
+    const json = localStorage.getItem(SETTINGS_KEY)
+    if (!json) return
+    let read
+    try { read = JSON.parse(json) }
+    catch {
+        console.error('invalid settings stored', json)
+        return
+    }
+    for (const k of SETTINGS_TO_STORE) {
+        const v = read[k]
+        if (v !== undefined) // @ts-ignore
+            state[k] = v
+    }
+}
+
+function storeSettings() {
+    localStorage.setItem(SETTINGS_KEY, JSON.stringify(_.pick(state, SETTINGS_TO_STORE)))
+}

package/frontend/src/useAuthorized.ts

@@ -0,0 +1,17 @@
+import { state, useSnapState } from './state'
+import { useNavigate } from 'react-router-dom'
+import { useEffect } from 'react'
+import { loginDialog } from './menu'
+
+export default function useAuthorized() {
+    const { loginRequired } = useSnapState()
+    const navigate = useNavigate()
+    useEffect(() => {
+        (async () => {
+            while (state.loginRequired)
+                await loginDialog(navigate).then()
+        })()
+    }, [loginRequired, navigate])
+    return loginRequired ? null : true
+}
+

package/frontend/src/useFetchList.ts

@@ -0,0 +1,144 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { state, useSnapState } from './state'
+import { useEffect, useRef } from 'react'
+import { apiEvents } from './api'
+import { DirEntry, DirList, usePath } from './BrowseFiles'
+import _ from 'lodash'
+import { subscribeKey } from 'valtio/utils'
+
+export default function useFetchList() {
+    const snap = useSnapState()
+    const desiredPath = usePath()
+    const search = snap.remoteSearch || undefined
+    const lastPath = useRef('')
+
+    useEffect(()=>{
+        const previous = lastPath.current
+        lastPath.current = desiredPath
+        if (previous !== desiredPath) {
+            state.showFilter = false
+            state.stopSearch?.()
+        }
+        state.stoppedSearch = false
+        if (previous !== desiredPath && search) {
+            state.remoteSearch = ''
+            return
+        }
+
+        const API = 'file_list'
+        const baseParams = { path:desiredPath, search, sse:true, omit:'c' }
+        state.list = []
+        state.filteredList = undefined
+        state.selected = {}
+        state.loading = true
+        state.error = undefined
+        // buffering entries is necessary against burst of events that will hang the browser
+        const buffer: DirList = []
+        const flush = () => {
+            const chunk = buffer.splice(0, Infinity)
+            if (chunk.length)
+                state.list = sort([...state.list, ...chunk.map(precalculate)])
+        }
+        const timer = setInterval(flush, 1000)
+        const src = apiEvents(API, baseParams, (type, data) => {
+            switch (type) {
+                case 'error':
+                    state.stopSearch?.()
+                    state.error = JSON.stringify(data)
+                    return
+                case 'closed':
+                    flush()
+                    state.stopSearch?.()
+                    state.loading = false
+                    return
+                case 'msg':
+                    data.forEach((data: any) => {
+                        if (data.add)
+                            return buffer.push(data.add)
+                        const { error } = data
+                        if (error === 405) { // "method not allowed" happens when we try to directly access an unauthorized file, and we get a login prompt, and then file_list the file (because we didn't know it was file or folder)
+                            state.messageOnly = "Your download should now start"
+                            window.location.reload() // reload will start the download, because now we got authenticated
+                            return
+                        }
+                        if (error) {
+                            state.stopSearch?.()
+                            state.error = (ERRORS as any)[error] || String(error)
+                            state.loginRequired = error === 401
+                            return
+                        }
+                    })
+                    if (src?.readyState === src?.CLOSED)
+                        return state.stopSearch?.()
+            }
+        })
+        state.stopSearch = ()=>{
+            state.stopSearch = undefined
+            buffer.length = 0
+            state.loading = false
+            clearInterval(timer)
+            src.close()
+        }
+    }, [desiredPath, search, snap.username, snap.listReloader])
+}
+
+const ERRORS = {
+    404: "Not found"
+}
+
+export function reloadList() {
+    state.listReloader = Date.now()
+}
+
+const { compare:localCompare } = new Intl.Collator(navigator.language)
+
+function sort(list: DirList) {
+    const { sortBy, foldersFirst } = state
+    // optimization: precalculate string comparisons
+    const bySize = sortBy === 'size'
+    const byExt = sortBy === 'extension'
+    const byTime = sortBy === 'time'
+    const invert = state.invertOrder ? -1 : 1
+    return list.sort((a,b) =>
+        foldersFirst && -compare(a.isFolder, b.isFolder)
+        || invert*(bySize ? compare(a.s||0, b.s||0)
+            : byExt ? localCompare(a.ext, b.ext)
+                : byTime ? compare(a.t, b.t)
+                    : 0
+        )
+        || invert*localCompare(a.n, b.n) // fallback to name/path
+    )
+}
+
+function precalculate(rec:DirEntry) {
+    const i = rec.n.lastIndexOf('.') + 1
+    rec.ext = i ? rec.n.substring(i) : ''
+    rec.isFolder = rec.n.endsWith('/')
+    const t = rec.m || rec.c
+    if (t)
+        rec.t = new Date(t)
+    return rec
+}
+
+// generic comparison
+function compare(a:any, b:any) {
+    return a < b ? -1 : a > b ? 1 : 0
+}
+
+// update list on sorting criteria
+const sortAgain = _.debounce(()=> state.list = sort(state.list), 100)
+subscribeKey(state, 'sortBy', sortAgain)
+subscribeKey(state, 'invertOrder', sortAgain)
+subscribeKey(state, 'foldersFirst', sortAgain)
+
+subscribeKey(state, 'patternFilter', v => {
+    if (!v)
+        return state.filteredList = undefined
+    const filter = new RegExp(_.escapeRegExp(v),'i')
+    const newList = []
+    for (const entry of state.list)
+        if (filter.test(entry.n))
+            newList.push(entry)
+    state.filteredList = newList
+})

package/frontend/src/useTheme.ts

@@ -0,0 +1,23 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { useSnapState } from './state'
+import { useEffect } from 'react'
+import { useDarkMode } from 'usehooks-ts'
+
+export default function useTheme() {
+    const { theme } = useSnapState()
+    const { isDarkMode } = useDarkMode()
+    useEffect(()=>{
+        const e = document.body
+        if (!e) return
+        const name = theme || (isDarkMode ? 'dark' : 'light')
+        const pre = 'theme-'
+        const ct = pre + name
+        const list = e.classList
+        for (const c of Array.from(list))
+            if (c.startsWith(pre) && c !== ct)
+                list.remove(c)
+        if (name)
+            list.add(ct)
+    }, [theme, isDarkMode])
+}

package/frontend/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "target": "es2017",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noFallthroughCasesInSwitch": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx"
+  },
+  "include": [
+    "src"
+  ]
+}

package/frontend/vite.config.ts

@@ -0,0 +1,21 @@
+import { defineConfig } from 'vite'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+    build: {
+        outDir: '../dist/frontend',
+        emptyOutDir: true,
+        target: "es2015",
+    },
+    server: {
+        port: 3005,
+        proxy: {
+            '/~/': {
+                target: 'http://localhost',
+                proxyTimeout: 2000,
+                changeOrigin: true,
+                ws: true,
+            }
+        }
+    }
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hfs",
-  "version": "0.26.7",
+  "version": "0.26.8",
   "description": "HTTP File Server",
   "keywords": [
     "file server",
@@ -24,6 +24,7 @@
     "build-shared": "npm run build --workspace=shared",
     "build-form": "npm run build --workspace=mui-grid-form",
     "server-for-test": "node dist/src --cwd . --config tests",
+    "server-for-test-dev": "cross-env DEV=1 nodemon --ignore tests/ --watch src -e ts,tsx --exec ts-node src -- --cwd . --config tests",
     "test": "mocha -r ts-node/register 'tests/**/*.ts'",
     "pub": "cd dist && npm publish",
     "dist": "npm run build-all && npm run dist-bin",

package/plugins/vhosting/plugin.js

@@ -21,7 +21,7 @@ exports.init = api => ({
     middleware(ctx) {
         let toModify = ctx
         if (ctx.path.startsWith(api.const.SPECIAL_URI)) { // special uris should be excluded...
-            toModify = ctx.request.query
+            toModify = ctx.params
             if (toModify.path === undefined) // ...unless they carry a path in the query. In that case we'll work that.
                 return
         }

package/src/QuickZipStream.ts

@@ -0,0 +1,279 @@
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+
+import { Readable } from 'stream'
+// @ts-ignore
+import { unsigned } from 'buffer-crc32'
+import assert from 'assert'
+
+const ZIP64_SIZE_LIMIT = 0xffffffff
+const ZIP64_NUMBER_LIMIT = 0xffff
+
+let crc32function: (input: string | Buffer, initialState?: number | undefined | null) => number
+import('@node-rs/crc32').then(lib => crc32function = lib.crc32, () => {
+    console.log('using generic lib for crc32')
+    crc32function = unsigned
+})
+
+const FLAGS = 0x0808 // bit3 = no crc in local header + bit11 = utf8
+
+interface ZipSource {
+    path: string
+    sourcePath?: string
+    getData: () => Readable // deferred stream, so that we don't keep many open files because of calculateSize()
+    size: number
+    ts: Date
+    mode?: number
+}
+export class QuickZipStream extends Readable {
+    private workingFile: Readable | undefined
+    private finished = false
+    private readonly entries: ({ size:number, crc:number, ts:Date, pathAsBuffer:Buffer, offset:number, version:number, extAttr: number })[] = []
+    private dataWritten = 0
+    private consumedCalculating: ZipSource[] = []
+    private skip: number = 0
+    private limit?: number
+
+    constructor(private readonly walker:  AsyncIterableIterator<ZipSource>) {
+        super({})
+    }
+
+    earlyClose() {
+        this.finished = true
+        this.push(null)
+    }
+
+    applyRange(start: number, end: number) {
+        if (end < start)
+            return this.earlyClose()
+        this.skip = start
+        this.limit = end - start + 1
+    }
+
+    controlledPush(chunk: number[] | Buffer) {
+        if (Array.isArray(chunk))
+            chunk = buffer(chunk)
+        this.dataWritten += chunk.length
+        if (this.skip) {
+            if (this.skip >= chunk.length) {
+                this.skip -= chunk.length
+                return true
+            }
+            chunk = chunk.subarray(this.skip)
+            this.skip = 0
+        }
+        const lastBit = this.limit! < chunk.length
+        if (lastBit)
+            chunk = chunk.subarray(0, this.limit)
+
+        const ret = this.push(chunk)
+        if (lastBit)
+            this.earlyClose()
+        return ret
+    }
+
+    async calculateSize(howLong:number = 1000) {
+        const endBy = Date.now() + howLong
+        while (1) {
+            if (Date.now() >= endBy)
+                return NaN
+            const { value } = await this.walker.next()
+            if (!value) break
+            this.consumedCalculating.push(value) // we keep same shape of the generator, so
+        }
+        // if we reach here, then we were able to consume all entries of the walker (in time)
+        let offset = 0
+        let centralDirSize = 0
+        for (const file of this.consumedCalculating) {
+            const pathSize = Buffer.from(file.path, 'utf8').length
+            const extraLength = (file.size > ZIP64_SIZE_LIMIT ? 2 : 0) + (offset > ZIP64_SIZE_LIMIT ? 1 : 0)
+            const extraDataSize = extraLength && (2+2 + extraLength*8)
+            offset += 4+2+2+2+ 4+4+4+4+ 2+2+ pathSize + file.size
+            centralDirSize += 4+2+2+2+2+ 4+4+4+4+ 2+2+2+2+2+ 4+4 + pathSize + extraDataSize
+        }
+        const n = this.consumedCalculating.length
+        const centralDirOffset = offset
+        if (n >= ZIP64_NUMBER_LIMIT
+        || centralDirOffset >= ZIP64_SIZE_LIMIT
+        || centralDirSize >= ZIP64_SIZE_LIMIT)
+            centralDirSize += 4+8+2+2+4+4+8+8+8+8+4+4+8+4
+        centralDirSize += 4+4+2+2+4+4+2
+        return offset + centralDirSize
+    }
+
+    async _read() {
+        if (this.finished || this.destroyed) return
+        if (this.workingFile)
+            return this.workingFile.resume()
+        const file = this.consumedCalculating.shift() || (await this.walker.next()).value as ZipSource
+        if (!file)
+            return this.closeArchive()
+        let { path, sourcePath, getData, size, ts, mode } = file
+        const pathAsBuffer = Buffer.from(path, 'utf8')
+        const offset = this.dataWritten
+        const version = 20
+        this.controlledPush([
+            4, 0x04034b50,
+            2, version,
+            2, FLAGS,
+            2, 0, // compression = store
+            ...ts2buf(ts),
+            4, 0, // crc
+            4, 0, // size
+            4, 0, // size
+            2, pathAsBuffer.length,
+            2, 0, // extra length
+        ])
+        this.controlledPush(pathAsBuffer)
+        if (this.finished) return
+
+        const cache = sourcePath ? crcCache[sourcePath] : undefined
+        const cacheHit = Number(cache?.ts) === Number(ts)
+        let crc = cacheHit ? cache!.crc : crc32function('')
+        const extAttr = !mode ? 0 : (mode | 0x8000) * 0x10000 // it's like <<16 but doesn't overflow so easily
+        const entry = { size, crc, pathAsBuffer, ts, offset, version, extAttr }
+        if (this.skip >= size && cacheHit) {
+            this.skip -= size
+            this.dataWritten += size
+            this.entries.push(entry)
+            setTimeout(() => this.push('')) // this "signal" works only after _read() is done
+            return
+        }
+        const data = getData()
+        data.on('error', (err) => console.error(err))
+        data.on('end', ()=>{
+            this.workingFile = undefined
+            entry.crc = crc
+            if (sourcePath)
+                crcCache[sourcePath] = { ts, crc }
+            this.entries.push(entry)
+            this.push('') // continue piping
+        })
+        this.workingFile = data
+        data.on('data', chunk => {
+            if (this.destroyed)
+                return data.destroy()
+            if (!this.controlledPush(chunk)) // destination buffer full
+                data.pause() // slow down
+            if (!cacheHit)
+                crc = crc32function(chunk, crc)
+            if (this.finished)
+                return data.destroy()
+        })
+    }
+
+    closeArchive() {
+        this.finished = true
+        let centralDirOffset = this.dataWritten
+        for (let { size, ts, crc, offset, pathAsBuffer, version, extAttr } of this.entries) {
+            const extra = []
+            if (size > ZIP64_SIZE_LIMIT) {
+                extra.push(size, size)
+                size = ZIP64_SIZE_LIMIT
+            }
+            if (offset > ZIP64_SIZE_LIMIT) {
+                extra.push(offset)
+                offset = ZIP64_SIZE_LIMIT
+            }
+            const extraData = buffer(!extra.length ? []
+                : [ 2,1, 2,8*extra.length, ...extra.map(x=> [8,x]).flat() ])
+            if (extraData.length && version < 45)
+                version = 45
+            this.controlledPush([
+                4, 0x02014b50, // central dir signature
+                2, version,
+                2, version,
+                2, FLAGS,
+                2, 0,    // compression method = store
+                ...ts2buf(ts),
+                4, crc,
+                4, size, // compressed
+                4, size,
+                2, pathAsBuffer.length,
+                2, extraData.length,
+                2, 0, //comment length
+                2, 0, // disk
+                2, 0, // attr
+                4, extAttr,
+                4, offset,
+            ])
+            this.controlledPush(pathAsBuffer)
+            this.controlledPush(extraData)
+        }
+        const after = this.dataWritten
+        let centralDirSize = after - centralDirOffset
+        let n = this.entries.length
+        if (n >= ZIP64_NUMBER_LIMIT
+            || centralDirOffset >= ZIP64_SIZE_LIMIT
+            || centralDirSize >= ZIP64_SIZE_LIMIT) {
+            this.controlledPush([
+                4, 0x06064b50, // end of central dir zip64
+                8, 44,
+                2, 45,
+                2, 45,
+                4, 0,
+                4, 0,
+                8, n,
+                8, n,
+                8, centralDirSize,
+                8, centralDirOffset,
+            ])
+            this.controlledPush([
+                4, 0x07064b50,
+                4, 0,
+                8, after,
+                4, 1,
+            ])
+            centralDirOffset = ZIP64_SIZE_LIMIT
+            centralDirSize = ZIP64_SIZE_LIMIT
+            n = ZIP64_NUMBER_LIMIT
+        }
+        this.controlledPush([
+            4, 0x06054b50, // end of central directory signature
+            4, 0, // disk-related stuff
+            2, n,
+            2, n,
+            4, centralDirSize,
+            4, centralDirOffset,
+            2, 0, // comment length
+        ])
+        this.push(null) // EOF
+    }
+}
+
+function buffer(pairs: number[]) {
+    assert(pairs.length % 2 === 0)
+    let total = 0
+    for (let i=0; i < pairs.length; i+=2)
+        total += pairs[i]
+    const ret = Buffer.alloc(total, 0)
+    let offset = 0
+    let i = 0
+    while (i < pairs.length) {
+        const size = pairs[i++]
+        const data = pairs[i++]
+        if (size === 1)
+            ret.writeUInt8(data, offset)
+        else if (size === 2)
+            ret.writeUInt16LE(data, offset)
+        else if (size === 4)
+            ret.writeUInt32LE(data, offset)
+        else if (size === 8)
+            ret.writeBigUInt64LE(BigInt(data), offset)
+        else
+            throw 'unsupported'
+        offset += size
+    }
+    return ret
+}
+
+function ts2buf(ts:Date) {
+    const date = ((ts.getFullYear() - 1980) & 0x7F) << 9 | (ts.getMonth() + 1) << 5 | ts.getDate()
+    const time = ts.getHours() << 11 | ts.getMinutes() << 5 | (ts.getSeconds() / 2) & 0x0F
+    return [
+        2, time,
+        2, date,
+    ]
+}
+
+interface CrcCacheEntry { ts: Date, crc: number }
+const crcCache: Record<string, CrcCacheEntry> = {}