hfs 0.26.8 → 0.26.9

package/src/serveGuiFiles.js

@@ -0,0 +1,113 @@
+"use strict";
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serveGuiFiles = void 0;
+const promises_1 = __importDefault(require("fs/promises"));
+const const_1 = require("./const");
+const serveFile_1 = require("./serveFile");
+const plugins_1 = require("./plugins");
+const api_auth_1 = require("./api.auth");
+const apiMiddleware_1 = require("./apiMiddleware");
+const path_1 = require("path");
+const misc_1 = require("./misc");
+// in case of dev env we have our static files within the 'dist' folder'
+const DEV_STATIC = process.env.DEV ? 'dist/' : '';
+function serveStatic(uri) {
+    const folder = uri.slice(2, -1); // we know folder is very similar to uri
+    const cache = {};
+    return async (ctx, next) => {
+        if (ctx.method === 'OPTIONS') {
+            ctx.status = const_1.NO_CONTENT;
+            ctx.set({ Allow: 'OPTIONS, GET' });
+            return;
+        }
+        if (ctx.method !== 'GET')
+            return ctx.status = const_1.METHOD_NOT_ALLOWED;
+        const serveApp = shouldServeApp(ctx);
+        const fullPath = (0, path_1.join)(__dirname, '..', DEV_STATIC, folder, serveApp ? '/index.html' : ctx.path);
+        const content = await (0, misc_1.getOrSet)(cache, ctx.path, async () => {
+            const data = await promises_1.default.readFile(fullPath).catch(() => null);
+            return serveApp || !data ? data
+                : adjustBundlerLinks(ctx.path, uri, data);
+        });
+        if (content === null)
+            return ctx.status = 404;
+        if (!serveApp)
+            return (0, serveFile_1.serveFile)(fullPath, 'auto', content)(ctx, next);
+        // we don't cache the index as it's small and may prevent plugins change to apply
+        ctx.body = await treatIndex(ctx, String(content), uri);
+        ctx.type = 'html';
+        ctx.set('Cache-Control', 'no-store, no-cache, must-revalidate');
+    };
+}
+function shouldServeApp(ctx) {
+    var _a;
+    return (_a = ctx.state).serveApp || (_a.serveApp = ctx.path.endsWith('/'));
+}
+function adjustBundlerLinks(path, uri, data) {
+    const ext = (0, path_1.extname)(path);
+    return ext && !ext.match(/\.(css|html|js|ts|scss)/) ? data
+        : String(data).replace(/((?:import | from )['"])\//g, `$1${uri}`);
+}
+async function treatIndex(ctx, body, filesUri) {
+    const session = await (0, api_auth_1.refresh_session)({}, ctx);
+    ctx.set('etag', '');
+    return body
+        .replace(/((?:src|href) *= *['"])\/?(?![a-z]+:\/\/)/g, '$1' + filesUri)
+        .replace('_HFS_SESSION_', session instanceof apiMiddleware_1.ApiError ? 'null' : JSON.stringify(session))
+        // replacing this text allow us to avoid injecting in frontends that don't support plugins. Don't use a <--comment--> or it will be removed by webpack
+        .replace('_HFS_PLUGINS_', pluginsInjection);
+}
+function serveProxied(port, uri) {
+    if (!port)
+        return;
+    console.debug('proxied on port', port);
+    let proxy;
+    Promise.resolve().then(() => __importStar(require('koa-better-http-proxy'))).then(lib => // dynamic import to avoid having this in final distribution
+     proxy = lib.default('127.0.0.1:' + port, {
+        proxyReqPathResolver: (ctx) => shouldServeApp(ctx) ? '/' : ctx.path,
+        userResDecorator(res, data, ctx) {
+            return shouldServeApp(ctx) ? treatIndex(ctx, String(data), uri)
+                : adjustBundlerLinks(ctx.path, uri, data);
+        }
+    }));
+    return function () {
+        return proxy.apply(this, arguments);
+    };
+}
+function pluginsInjection() {
+    const css = (0, plugins_1.mapPlugins)((plug, k) => { var _a; return (_a = plug.frontend_css) === null || _a === void 0 ? void 0 : _a.map(f => const_1.PLUGINS_PUB_URI + k + '/' + f); }).flat().filter(Boolean);
+    const js = (0, plugins_1.mapPlugins)((plug, k) => { var _a; return (_a = plug.frontend_js) === null || _a === void 0 ? void 0 : _a.map(f => const_1.PLUGINS_PUB_URI + k + '/' + f); }).flat().filter(Boolean);
+    return css.map(uri => `\n<link rel='stylesheet' type='text/css' href='${uri}'/>`).join('')
+        + js.map(uri => `\n<script defer src='${uri}'></script>`).join('');
+}
+function serveGuiFiles(proxyPort, uri) {
+    return serveProxied(proxyPort, uri) || serveStatic(uri);
+}
+exports.serveGuiFiles = serveGuiFiles;

package/src/sse.js

@@ -0,0 +1,29 @@
+"use strict";
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_1 = require("stream");
+function createSSE(ctx) {
+    const { socket } = ctx.req;
+    socket.setTimeout(0);
+    socket.setNoDelay(true);
+    socket.setKeepAlive(true);
+    ctx.set({
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'X-Accel-Buffering': 'no', // avoid buffering when reverse-proxied through nginx
+    });
+    ctx.status = 200;
+    return ctx.body = new stream_1.Transform({
+        objectMode: true,
+        transform(chunk, encoding, cb) {
+            this.push(`data: ${JSON.stringify(chunk)}\n\n`);
+            cb();
+        },
+        flush(cb) {
+            this.push('data:\n\n');
+            cb();
+        }
+    });
+}
+exports.default = createSSE;

package/src/throttler.js

@@ -0,0 +1,91 @@
+"use strict";
+// This file is part of HFS - Copyright 2021-2022, Massimo Melina <a@rejetto.com> - License https://www.gnu.org/licenses/gpl-3.0.txt
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.totalInSpeed = exports.totalOutSpeed = exports.totalGot = exports.totalSent = exports.throttler = void 0;
+const stream_1 = require("stream");
+const ThrottledStream_1 = require("./ThrottledStream");
+const config_1 = require("./config");
+const misc_1 = require("./misc");
+const connections_1 = require("./connections");
+const lodash_1 = __importDefault(require("lodash"));
+const events_1 = __importDefault(require("./events"));
+const mainThrottleGroup = new ThrottledStream_1.ThrottleGroup(Infinity);
+(0, config_1.defineConfig)('max_kbps', Infinity).sub(v => mainThrottleGroup.updateLimit(v));
+const ip2group = {};
+const SymThrStr = Symbol('stream');
+const SymTimeout = Symbol('timeout');
+const maxKbpsPerIp = (0, config_1.defineConfig)('max_kbps_per_ip', Infinity);
+const throttler = async (ctx, next) => {
+    await next();
+    const { body } = ctx;
+    if (!body || !(body instanceof stream_1.Readable))
+        return;
+    // we wrap the stream also for unlimited connections to get speed and other features
+    const ipGroup = (0, misc_1.getOrSet)(ip2group, ctx.ip, () => {
+        var _a;
+        const doLimit = ((_a = ctx.state.account) === null || _a === void 0 ? void 0 : _a.ignore_limits) || (0, misc_1.isLocalHost)(ctx) ? undefined : true;
+        const group = new ThrottledStream_1.ThrottleGroup(Infinity, doLimit && mainThrottleGroup);
+        const unsub = doLimit && maxKbpsPerIp.sub(v => group.updateLimit(v));
+        return { group, count: 0, destroy: unsub };
+    });
+    const conn = ctx.state.connection;
+    if (!conn)
+        throw 'assert throttler connection';
+    const ts = conn[SymThrStr] = new ThrottledStream_1.ThrottledStream(ipGroup.group, conn[SymThrStr]);
+    let closed = false;
+    const DELAY = 1000;
+    const update = lodash_1.default.debounce(() => {
+        const ts = conn[SymThrStr];
+        const outSpeed = roundKb(ts.getSpeed());
+        (0, connections_1.updateConnection)(conn, { outSpeed, sent: ts.getBytesSent() });
+        /* in case this stream stands still for a while (before the end), we'll have neither 'sent' or 'close' events,
+        * so who will take care to updateConnection? This artificial next-call will ensure just that */
+        clearTimeout(conn[SymTimeout]);
+        if (outSpeed || !closed)
+            conn[SymTimeout] = setTimeout(update, DELAY);
+    }, DELAY, { maxWait: DELAY });
+    ts.on('sent', (n) => {
+        exports.totalSent += n;
+        update();
+    });
+    ++ipGroup.count;
+    ts.on('close', () => {
+        var _a;
+        update.flush();
+        closed = true;
+        if (--ipGroup.count)
+            return; // any left?
+        (_a = ipGroup.destroy) === null || _a === void 0 ? void 0 : _a.call(ipGroup);
+        delete ip2group[ctx.ip];
+    });
+    const bak = ctx.response.length; // preserve
+    ctx.body = ctx.body.pipe(ts);
+    if (bak)
+        ctx.response.length = bak;
+};
+exports.throttler = throttler;
+function roundKb(n) {
+    return lodash_1.default.round(n, 1) || lodash_1.default.round(n, 3); // further precision if necessary
+}
+exports.totalSent = 0;
+exports.totalGot = 0;
+exports.totalOutSpeed = 0;
+exports.totalInSpeed = 0;
+let lastSent = exports.totalSent;
+let lastGot = exports.totalGot;
+let last = Date.now();
+setInterval(() => {
+    const now = Date.now();
+    const past = (now - last) / 1000; // seconds
+    last = now;
+    const deltaSentKb = (exports.totalSent - lastSent) / 1000;
+    lastSent = exports.totalSent;
+    const deltaGotKb = (exports.totalGot - lastGot) / 1000;
+    lastGot = exports.totalGot;
+    exports.totalOutSpeed = roundKb(deltaSentKb / past);
+    exports.totalInSpeed = roundKb(deltaGotKb / past);
+}, 1000);
+events_1.default.on('connection', (c) => c.socket.on('data', data => exports.totalGot += data.length));

package/src/update.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.update = exports.getUpdate = void 0;
+const github_1 = require("./github");
+const const_1 = require("./const");
+const path_1 = require("path");
+const child_process_1 = require("child_process");
+const misc_1 = require("./misc");
+const fs_1 = require("fs");
+const plugins_1 = require("./plugins");
+const promises_1 = require("fs/promises");
+const HFS_REPO = 'rejetto/hfs';
+async function getUpdate() {
+    const [latest] = await (0, github_1.getRepoInfo)(HFS_REPO + '/releases?per_page=1');
+    if (latest.name === const_1.VERSION)
+        throw "you already have the latest version: " + const_1.VERSION;
+    return latest;
+}
+exports.getUpdate = getUpdate;
+async function update() {
+    if (process.argv0.endsWith('node'))
+        throw "only binary versions are supported for now";
+    const update = await getUpdate();
+    const assetSearch = { win32: 'windows', darwin: 'mac', linux: 'linux' }[process.platform];
+    if (!assetSearch)
+        throw "this feature doesn't support your platform: " + process.platform;
+    const asset = update.assets.find((x) => x.name.endsWith('.zip') && x.name.includes(assetSearch));
+    if (!asset)
+        throw "asset not found";
+    const url = asset.browser_download_url;
+    console.log("downloading", url);
+    const bin = process.argv[0];
+    const binPath = (0, path_1.dirname)(bin);
+    const binFile = (0, path_1.basename)(bin);
+    const newBinFile = 'new-' + binFile;
+    plugins_1.pluginsWatcher.pause();
+    try {
+        await (0, misc_1.unzip)(await (0, misc_1.httpsStream)(url), path => (0, path_1.join)(binPath, path === binFile ? newBinFile : path));
+        const newBin = (0, path_1.join)(binPath, newBinFile);
+        if (!const_1.IS_WINDOWS) {
+            const { mode } = await (0, promises_1.stat)(bin);
+            await (0, promises_1.chmod)(newBin, mode).catch(console.error);
+        }
+        (0, misc_1.onProcessExit)(() => {
+            const oldBinFile = 'old-' + binFile;
+            console.log("old version is kept as", oldBinFile);
+            const oldBin = (0, path_1.join)(binPath, oldBinFile);
+            try {
+                (0, fs_1.unlinkSync)(oldBin);
+            }
+            catch (_a) { }
+            (0, fs_1.renameSync)(bin, oldBin);
+            console.log("launching new version in background", newBinFile);
+            (0, child_process_1.spawn)(newBin, ['--updating', binFile], { detached: true, shell: true })
+                .on('error', console.error);
+        });
+        console.log('quitting');
+        process.exit();
+    }
+    catch (_a) {
+        plugins_1.pluginsWatcher.unpause();
+    }
+}
+exports.update = update;
+if (const_1.argv.updating) { // we were launched with a temporary name, restore original name to avoid breaking references
+    const bin = process.argv[0];
+    (0, fs_1.renameSync)(bin, (0, path_1.join)((0, path_1.dirname)(bin), const_1.argv.updating));
+    console.log("renamed binary file to", const_1.argv.updating);
+}

package/src/util-files.js

@@ -0,0 +1,148 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unzip = exports.run = exports.dirStream = exports.adjustStaticPathForGlob = exports.isWindowsDrive = exports.dirTraversal = exports.watchDir = exports.readFileBusy = exports.isFile = exports.isDirectory = void 0;
+const promises_1 = __importDefault(require("fs/promises"));
+const misc_1 = require("./misc");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const fast_glob_1 = __importDefault(require("fast-glob"));
+const const_1 = require("./const");
+const child_process_1 = require("child_process");
+const stream_1 = require("stream");
+// @ts-ignore
+const unzip_stream_1 = __importDefault(require("unzip-stream"));
+async function isDirectory(path) {
+    try {
+        return (await promises_1.default.stat(path)).isDirectory();
+    }
+    catch (_a) {
+        return false;
+    }
+}
+exports.isDirectory = isDirectory;
+async function isFile(path) {
+    try {
+        return (await promises_1.default.stat(path)).isFile();
+    }
+    catch (_a) {
+        return false;
+    }
+}
+exports.isFile = isFile;
+async function readFileBusy(path) {
+    return promises_1.default.readFile(path, 'utf8').catch(e => {
+        if ((e === null || e === void 0 ? void 0 : e.code) !== 'EBUSY')
+            throw e;
+        console.debug('busy');
+        return (0, misc_1.wait)(100).then(() => readFileBusy(path));
+    });
+}
+exports.readFileBusy = readFileBusy;
+function watchDir(dir, cb) {
+    let watcher;
+    let paused = false;
+    try {
+        watcher = (0, fs_1.watch)(dir, controlledCb);
+    }
+    catch (_a) {
+        // failing watching the content of the dir, we try to monitor its parent, but filtering events only for our target dir
+        const base = (0, path_1.basename)(dir);
+        try {
+            watcher = (0, fs_1.watch)((0, path_1.dirname)(dir), (event, name) => {
+                if (name !== base)
+                    return;
+                try {
+                    watcher.close(); // if we succeed, we give up the parent watching
+                    watcher = (0, fs_1.watch)(dir, controlledCb); // attempt at passing to a more specific watching
+                }
+                catch (_a) { }
+                controlledCb();
+            });
+        }
+        catch (e) {
+            console.debug(String(e));
+        }
+    }
+    return {
+        working() { return Boolean(watcher); },
+        stop() { watcher === null || watcher === void 0 ? void 0 : watcher.close(); },
+        pause() { paused = true; },
+        unpause() { paused = false; },
+    };
+    function controlledCb() {
+        if (!paused)
+            cb();
+    }
+}
+exports.watchDir = watchDir;
+function dirTraversal(s) {
+    return s && /(^|[/\\])\.\.($|[/\\])/.test(s);
+}
+exports.dirTraversal = dirTraversal;
+function isWindowsDrive(s) {
+    return s && /^[a-zA-Z]:$/.test(s);
+}
+exports.isWindowsDrive = isWindowsDrive;
+// apply this to paths that may contain \ as separator (not supported by fast-glob) and other special chars to be escaped (parenthesis)
+function adjustStaticPathForGlob(path) {
+    return fast_glob_1.default.escapePath(path.replace(/\\/g, '/'));
+}
+exports.adjustStaticPathForGlob = adjustStaticPathForGlob;
+async function* dirStream(path, deep) {
+    if (!await isDirectory(path))
+        throw Error('ENOTDIR');
+    const dirStream = fast_glob_1.default.stream(deep ? '**/*' : '*', {
+        cwd: path,
+        dot: true,
+        deep,
+        onlyFiles: false,
+        suppressErrors: true,
+        objectMode: true,
+    });
+    const skip = await getItemsToSkip(path);
+    for await (const entry of dirStream) {
+        let { path, dirent } = entry;
+        if (!dirent.isDirectory() && !dirent.isFile())
+            continue;
+        path = String(path);
+        if (!(skip === null || skip === void 0 ? void 0 : skip.includes(path)))
+            yield path;
+    }
+    async function getItemsToSkip(path) {
+        if (!const_1.IS_WINDOWS)
+            return;
+        const out = await run('dir', ['/ah', '/b', path.replace(/\//g, '\\')])
+            .catch(() => ''); // error in case of no matching file
+        return out.split('\r\n').slice(0, -1);
+    }
+}
+exports.dirStream = dirStream;
+function run(cmd, args = []) {
+    return new Promise((resolve, reject) => (0, child_process_1.execFile)('cmd', ['/c', cmd, ...args], (err, stdout) => {
+        if (err)
+            reject(err);
+        else
+            resolve(stdout);
+    }));
+}
+exports.run = run;
+async function unzip(stream, cb) {
+    let pending = Promise.resolve();
+    return new Promise(resolve => stream.pipe(unzip_stream_1.default.Parse())
+        .on('end', () => pending.then(resolve))
+        .on('entry', async (entry) => {
+        const { path, type } = entry;
+        const dest = cb(path);
+        if (!dest || type !== 'File')
+            return entry.autodrain();
+        await pending; // don't overlap writings
+        console.debug('unzip', dest);
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(dest), { recursive: true }); // easy way be sure to have the folder ready before proceeding
+        const thisFile = entry.pipe((0, fs_1.createWriteStream)(dest));
+        pending = (0, stream_1.once)(thisFile, 'finish');
+    }));
+}
+exports.unzip = unzip;

package/src/util-generators.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.asyncGeneratorToReadable = exports.asyncGeneratorToArray = exports.filterMapGenerator = void 0;
+// callback can return undefined to skip element
+const stream_1 = require("stream");
+async function* filterMapGenerator(generator, filterMap) {
+    for await (const x of generator) {
+        const res = await filterMap(x);
+        if (res !== undefined)
+            yield res;
+    }
+}
+exports.filterMapGenerator = filterMapGenerator;
+async function asyncGeneratorToArray(generator) {
+    const ret = [];
+    for await (const x of generator)
+        ret.push(x);
+    return ret;
+}
+exports.asyncGeneratorToArray = asyncGeneratorToArray;
+function asyncGeneratorToReadable(generator) {
+    const iterator = generator[Symbol.asyncIterator]();
+    return new stream_1.Readable({
+        objectMode: true,
+        read() {
+            iterator.next().then(it => this.push(it.done ? null : it.value));
+        }
+    });
+}
+exports.asyncGeneratorToReadable = asyncGeneratorToReadable;

package/src/util-http.js

@@ -0,0 +1,30 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.httpsStream = exports.httpsString = void 0;
+const node_https_1 = __importDefault(require("node:https"));
+function httpsString(url, options = {}) {
+    return httpsStream(url, options).then(res => new Promise(resolve => {
+        let buf = '';
+        res.on('data', chunk => buf += chunk.toString());
+        res.on('end', () => resolve(Object.assign(res, {
+            ok: (res.statusCode || 400) < 400,
+            body: buf
+        })));
+    }));
+}
+exports.httpsString = httpsString;
+function httpsStream(url, options = {}) {
+    return new Promise((resolve, reject) => {
+        node_https_1.default.request(url, options, res => {
+            if (!res.statusCode || res.statusCode >= 400)
+                throw res;
+            if (res.statusCode === 302 && res.headers.location)
+                return resolve(httpsStream(res.headers.location, options));
+            resolve(res);
+        }).on('error', reject).end();
+    });
+}
+exports.httpsStream = httpsStream;