hfs 3.2.2 → 3.3.0-beta2

package/src/throttler.js

@@ -15,11 +15,11 @@ const lodash_1 = __importDefault(require("lodash"));
 const events_1 = __importDefault(require("./events"));
 const persistence_1 = require("./persistence");
 const mainThrottleGroup = new ThrottledStream_1.ThrottleGroup(Infinity);
-(0, config_1.defineConfig)('max_kbps', Infinity).sub(v => mainThrottleGroup.updateLimit(v));
+(0, config_1.defineConfig)(misc_1.CFG.max_kbps, Infinity).sub(v => mainThrottleGroup.updateLimit(v));
 const ip2group = {};
 const SymThrStr = Symbol('stream');
 const SymTimeout = Symbol('timeout');
-const maxKbpsPerIp = (0, config_1.defineConfig)('max_kbps_per_ip', Infinity);
+const maxKbpsPerIp = (0, config_1.defineConfig)(misc_1.CFG.max_kbps_per_ip, Infinity);
 maxKbpsPerIp.sub(v => {
     for (const [ip, { group }] of Object.entries(ip2group))
         if (ip) // empty-string = unlimited group
@@ -71,6 +71,7 @@ const throttler = async (ctx, next) => {
         delete ip2group[ctx.ip];
     });
     ctx.state.originalStream = body;
+    body.on('error', e => ts.destroy(e)); // pipe doesn't forward source errors, so make Koa's response stream report them
     ctx.body = body.pipe(ts);
     if (downloadTotal !== undefined) // undefined will break SSE
         ctx.response.length = downloadTotal; // preserve this info

package/src/update.js

@@ -28,8 +28,8 @@ const persistence_1 = require("./persistence");
 const lodash_1 = __importDefault(require("lodash"));
 const argv_1 = require("./argv");
 const promises_2 = require("stream/promises");
-const updateToBeta = (0, config_1.defineConfig)('update_to_beta', false);
-const autoCheckUpdate = (0, config_1.defineConfig)('auto_check_update', true);
+const updateToBeta = (0, config_1.defineConfig)(misc_1.CFG.update_to_beta, false);
+const autoCheckUpdate = (0, config_1.defineConfig)(misc_1.CFG.auto_check_update, true);
 const lastCheckUpdate = persistence_1.storedMap.singleSync('lastCheckUpdate', 0);
 const AUTO_CHECK_EVERY = misc_1.DAY;
 exports.autoCheckUpdateResult = persistence_1.storedMap.singleSync('autoCheckUpdateResult', undefined);
@@ -162,7 +162,13 @@ async function update(tagOrUrl = '') {
     } while ((0, fs_1.existsSync)((0, path_1.join)(binPath, newBinFile)));
     plugins_1.pluginsWatcher.pause();
     try {
-        await (0, misc_1.unzip)((0, fs_1.createReadStream)(LOCAL_UPDATE), path => (0, path_1.join)(binPath, path === binFile ? newBinFile : path));
+        try {
+            await (0, misc_1.unzip)((0, fs_1.createReadStream)(LOCAL_UPDATE), path => (0, path_1.join)(binPath, path === binFile ? newBinFile : path));
+        }
+        catch (e) {
+            await (0, promises_1.rename)(LOCAL_UPDATE, 'hfs-update-bad.zip'); // quarantine failed archives so they don't take precedence over future downloads
+            throw e;
+        }
         const newBin = (0, path_1.join)(binPath, newBinFile);
         if (!(0, fs_1.existsSync)(newBin)) {
             if (url) // the file was downloaded, and the UI would show the "update from local file" button until we remove it

package/src/upload.js

@@ -23,10 +23,15 @@ const events_1 = __importDefault(require("./events"));
 const promises_1 = require("fs/promises");
 const expiringCache_1 = require("./expiringCache");
 const first_1 = require("./first");
-exports.deleteUnfinishedUploadsAfter = (0, config_1.defineConfig)('delete_unfinished_uploads_after', 86_400);
-exports.minAvailableMb = (0, config_1.defineConfig)('min_available_mb', 100);
-exports.dontOverwriteUploading = (0, config_1.defineConfig)('dont_overwrite_uploading', true);
+const uploadOwners_1 = require("./uploadOwners");
+exports.deleteUnfinishedUploadsAfter = (0, config_1.defineConfig)(misc_1.CFG.delete_unfinished_uploads_after, 86_400);
+exports.minAvailableMb = (0, config_1.defineConfig)(misc_1.CFG.min_available_mb, 100);
+exports.dontOverwriteUploading = (0, config_1.defineConfig)(misc_1.CFG.dont_overwrite_uploading, true);
 const waitingToBeDeleted = {};
+function cancelDeletion(path) {
+    clearTimeout(waitingToBeDeleted[path]?.timeout);
+    delete waitingToBeDeleted[path];
+}
 (0, first_1.onProcessExit)(() => {
     if (!Object.keys(waitingToBeDeleted).length)
         return;
@@ -48,7 +53,7 @@ function setUploadMeta(path, ctx) {
     });
 }
 function getUploadTempFor(fullPath) {
-    return (0, path_1.join)((0, path_1.dirname)(fullPath), 'hfs$upload-' + (0, path_1.basename)(fullPath).slice(-200));
+    return (0, path_1.join)((0, path_1.dirname)(fullPath), const_1.UPLOAD_TEMP_PREFIX + (0, path_1.basename)(fullPath).slice(-200));
 }
 const diskSpaceCache = (0, expiringCache_1.expiringCache)(3_000); // invalidate shortly
 const uploadingFiles = new Map();
@@ -105,6 +110,7 @@ function uploadWriter(base, baseUri, filename, ctx) {
             setUploadMeta(dir, ctx);
         // use temporary name while uploading
         const tempName = getUploadTempFor(fullPath);
+        const tempUri = (0, misc_1.enforceFinal)('/', baseUri) + (0, misc_1.pathEncode)((0, path_1.basename)(tempName));
         // try to catch errors early (sync): this is avoiding on chrome139 when uploading a big file (1GB) to get miss the error code and have to make a `simulate`
         try {
             fs_1.default.accessSync(tempName, fs_1.default.constants.W_OK);
@@ -130,8 +136,11 @@ function uploadWriter(base, baseUri, filename, ctx) {
                 return fail(const_1.HTTP_PRECONDITION_FAILED);
             }
         // append if resuming
-        if (!resume && stats)
+        if (!resume && stats) {
+            // a new upload discards the old resumable temp, so its owner grant must not survive
+            (0, uploadOwners_1.deleteUploadOwner)(tempUri);
             fs_1.default.unlinkSync(tempName);
+        }
         const writeStream = (0, misc_1.createStreamLimiter)(isNaN(contentLength) ? Infinity : contentLength);
         const fullSize = stillToWrite + resume;
         // allow plugins to mess with the write-stream, because the read-stream can be complicated in case of multipart
@@ -152,6 +161,9 @@ function uploadWriter(base, baseUri, filename, ctx) {
         const tracked = { ctx, got: 0, size: stillToWrite };
         uploadingFiles.set(fullPath, tracked);
         console.debug('Upload started');
+        let tempOwnerSet = false;
+        // ownership must be visible before a follow-up request can act on the unfinished file
+        ctx.req.once('aborted', setTempOwner);
         // the file stream doesn't have an event for data being written, so we use 'data' of its feeder, which happens before, so we postpone a bit, trying to have a fresher number
         writeStream.on('data', () => setTimeout(() => tracked.got = bytesGot()));
         const lockMiddleware = (0, misc_1.pendingPromise)(); // expose outside, to let know when all operations stopped
@@ -171,7 +183,8 @@ function uploadWriter(base, baseUri, filename, ctx) {
                     return (0, promises_1.rm)(tempName).catch(() => { });
                 if (ctx.isAborted()) { // in the very unlikely case the connection is interrupted between last-byte and here, we still consider it unfinished, as the client had no way to know, and will resume, but it would get an error if we finish the process
                     const sec = exports.deleteUnfinishedUploadsAfter.get();
-                    return lodash_1.default.isNumber(sec) && delayedDelete(tempName, sec);
+                    setTempOwner();
+                    return lodash_1.default.isNumber(sec) && delayedDelete(tempName, sec, tempUri);
                 }
                 if (isPartial) // we are supposed to leave the unfinished upload as it is, with its temp name
                     return ctx.status = const_1.HTTP_NO_CONTENT; // lockMiddleware contains an empty string, so we must take care of the status
@@ -194,6 +207,7 @@ function uploadWriter(base, baseUri, filename, ctx) {
                     () => (0, promises_1.rename)(tempName, dest).then(() => true, e => e?.code !== 'EBUSY' && Promise.reject(e)), { timeout: 10_000 });
                     if (!done)
                         throw 'EBUSY';
+                    (0, uploadOwners_1.deleteUploadOwner)(tempUri); // the temp URI no longer exists after rename; final ownership is recorded below
                     if (mtime) // so we use it to touch the file
                         await (0, promises_1.utimes)(dest, Date.now() / 1000, mtime / 1000);
                     cancelDeletion(tempName); // not necessary, as deletion's failure is silent, but still
@@ -202,6 +216,7 @@ function uploadWriter(base, baseUri, filename, ctx) {
                     if (ctx.query.comment)
                         void (0, comments_1.setCommentFor)(dest, String(ctx.query.comment));
                     obj.uri = (0, misc_1.enforceFinal)('/', baseUri) + (0, misc_1.pathEncode)((0, path_1.basename)(dest));
+                    await (0, uploadOwners_1.setUploadOwner)(obj.uri, ctx);
                     events_1.default.emit('uploadFinished', obj);
                     console.debug("Upload finished", dest);
                     if (resEvent)
@@ -245,6 +260,12 @@ function uploadWriter(base, baseUri, filename, ctx) {
         function bytesGot() {
             return fileStream.bytesWritten + fileStream.writableLength;
         }
+        function setTempOwner() {
+            if (tempOwnerSet)
+                return;
+            tempOwnerSet = true;
+            void (0, uploadOwners_1.setUploadOwner)(tempUri, ctx);
+        }
     }
     catch (e) {
         releaseFile();
@@ -259,21 +280,18 @@ function uploadWriter(base, baseUri, filename, ctx) {
         overwriteRequestedButForbidden = true;
         return false;
     }
-    function delayedDelete(path, secs) {
+    function delayedDelete(path, secs, tempUri) {
         clearTimeout(waitingToBeDeleted[path]?.timeout);
         return waitingToBeDeleted[path] = {
             mtime,
             expires: Date.now() + secs * 1000,
             timeout: setTimeout(() => {
                 delete waitingToBeDeleted[path];
+                (0, uploadOwners_1.deleteUploadOwner)(tempUri);
                 (0, promises_1.rm)(path).catch(() => { });
             }, secs * 1000)
         };
     }
-    function cancelDeletion(path) {
-        clearTimeout(waitingToBeDeleted[path]?.timeout);
-        delete waitingToBeDeleted[path];
-    }
     function releaseFile() {
         uploadingFiles.delete(fullPath);
     }

package/src/uploadOwners.js

@@ -0,0 +1,95 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadOwners = exports.ownUploadDeleteHours = void 0;
+exports.setUploadOwner = setUploadOwner;
+exports.moveUploadOwner = moveUploadOwner;
+exports.deleteUploadOwner = deleteUploadOwner;
+exports.getSessionId = getSessionId;
+const kvstorage_1 = require("@rejetto/kvstorage");
+const misc_1 = require("./misc");
+const first_1 = require("./first");
+const config_1 = require("./config");
+const auth_1 = require("./auth");
+const events_1 = __importDefault(require("./events"));
+exports.ownUploadDeleteHours = (0, config_1.defineConfig)(misc_1.CFG.own_upload_delete_hours, 24);
+exports.uploadOwners = new kvstorage_1.KvStorage({
+    rewriteLater: true,
+});
+exports.uploadOwners.open('upload-owners.kv').catch(e => console.error("Upload owners won't work correctly", e));
+(0, first_1.onProcessExit)(() => exports.uploadOwners.close());
+setInterval(() => {
+    if (!exports.uploadOwners.isOpen())
+        return;
+    const keys = Array.from(exports.uploadOwners.keys());
+    for (const k of keys) {
+        const owner = exports.uploadOwners.getSync(k);
+        if (owner && isExpired(owner))
+            void exports.uploadOwners.del(k);
+    }
+}, misc_1.MINUTE);
+events_1.default.on('checkVfsPermission', ({ node, perm, ctx }) => {
+    if (perm !== 'can_delete' || !node.source)
+        return;
+    const { vfsPath } = node;
+    if (!vfsPath || !exports.ownUploadDeleteHours.get() || !exports.uploadOwners.isOpen())
+        return;
+    const owner = exports.uploadOwners.getSync(cleanVfsPath(vfsPath));
+    if (!owner)
+        return;
+    if (isExpired(owner)) {
+        deleteUploadOwner(vfsPath);
+        return;
+    }
+    const username = (0, auth_1.getCurrentUsername)(ctx);
+    const { sessionId } = owner;
+    if (sessionId && sessionId === ctx.session?.sessionId || username && owner.username === username)
+        return 0;
+});
+function setUploadOwner(vfsPath, ctx) {
+    if (!exports.uploadOwners.isOpen() || !exports.ownUploadDeleteHours.get())
+        return;
+    const username = (0, auth_1.getCurrentUsername)(ctx) || undefined;
+    return exports.uploadOwners.put(cleanVfsPath(vfsPath), {
+        username,
+        sessionId: username ? undefined : getSessionId(ctx),
+        ip: ctx.ip,
+        created: new Date(),
+    })?.catch(e => {
+        console.error("Couldn't store upload owner for", vfsPath, String(e.message || e));
+    });
+}
+async function moveUploadOwner(fromPath, toPath) {
+    if (!exports.uploadOwners.isOpen())
+        return;
+    const from = cleanVfsPath(fromPath);
+    const affected = Array.from(exports.uploadOwners.keys()).filter(k => isSameOrInside(from, k));
+    if (!affected.length)
+        return;
+    const to = cleanVfsPath(toPath);
+    const owners = affected.map(k => ({ k, owner: exports.uploadOwners.getSync(k) }));
+    // ownership is keyed by VFS path, so HFS moves must carry descendant upload records too
+    await Promise.all(owners.map(({ k, owner }) => owner && exports.uploadOwners.put(to + k.slice(from.length), owner)));
+    await Promise.all(affected.map(k => exports.uploadOwners.del(k)));
+}
+function deleteUploadOwner(vfsPath) {
+    if (exports.uploadOwners.isOpen())
+        // deleting a folder must clear ownership for uploaded files below it too
+        for (const k of Array.from(exports.uploadOwners.keys()).filter(k => isSameOrInside(cleanVfsPath(vfsPath), k)))
+            void exports.uploadOwners.del(k);
+}
+function isExpired(owner) {
+    const hours = exports.ownUploadDeleteHours.get();
+    return !hours || Number(owner.created) + hours * misc_1.HOUR <= Date.now();
+}
+function getSessionId(ctx) {
+    return ctx.session.sessionId ||= (0, misc_1.randomId)(30);
+}
+function cleanVfsPath(path) {
+    return '/' + path.replace(/^\/+|\/+$/g, '');
+}
+function isSameOrInside(parent, path) {
+    return path === parent || path.startsWith(parent + '/');
+}

package/src/util-files.js

@@ -31,7 +31,7 @@ const const_1 = require("./const");
 const promises_2 = require("stream/promises");
 const stat_1 = require("./stat");
 const unzipper_1 = __importDefault(require("unzipper"));
-const fileTimeout = (0, config_1.defineConfig)('file_timeout', 3, x => x * 1000);
+const fileTimeout = (0, config_1.defineConfig)(cross_1.CFG.file_timeout, 3, x => x * 1000);
 // a smart (and a bit arbitrary) way to decide if we need the stat-workers functionality. Without it, we may be a bit faster. We'll see with experience if we need a dedicated configuration.
 const disableStatWorkers = Number(process.env.UV_THREADPOOL_SIZE) >= 10;
 // some paths (mostly offline networked ones) can take 20+ seconds to fail
@@ -106,7 +106,9 @@ function escapeGlobPath(path) {
 async function unzip(stream, cb) {
     const extracted = new Map();
     let chain = Promise.resolve();
-    return new Promise((resolve, reject) => stream.pipe(unzipper_1.default.Parse())
+    const parser = unzipper_1.default.Parse();
+    stream.once('error', e => parser.destroy(e)); // pipe doesn't forward source errors, so make the parser reject and release its resources
+    return new Promise((resolve, reject) => stream.pipe(parser)
         .on('close', () => chain.then(resolve, reject))
         .on('error', reject)
         .on('entry', (entry) => chain = chain.then(async () => {

package/src/util-http.js

@@ -64,6 +64,7 @@ async function httpWithBody(url, options) {
 function httpStream(url, { body, proxy, jar, noRedirect, httpThrow = true, ...options } = {}, redirected = []) {
     const controller = new AbortController();
     options.signal ??= controller.signal;
+    options.localAddress ??= httpStream.defaultLocalAddress;
     return Object.assign(new Promise(async (resolve, reject) => {
         proxy ??= httpStream.defaultProxy;
         options.headers ??= {};
@@ -140,7 +141,7 @@ function httpStream(url, { body, proxy, jar, noRedirect, httpThrow = true, ...op
         if (options.timeout) // node only emits the timeout event, so destroy the request to unblock callers waiting for the body
             req.setTimeout(options.timeout, () => req.destroy(Object.assign(Error('timeout'), { code: 'ETIMEDOUT' })));
         if (body && body instanceof node_stream_1.Readable)
-            body.pipe(req).on('end', () => req.end());
+            body.on('error', e => req.destroy(e)).pipe(req);
         else
             req.end(body);
         function connect() {
@@ -149,6 +150,7 @@ function httpStream(url, { body, proxy, jar, noRedirect, httpThrow = true, ...op
                 (proxyParsed.protocol === 'https:' ? node_https_1.default : node_http_1.default).request({
                     ...proxyParsed,
                     auth: undefined, // void proxyParsed.auth
+                    localAddress: options.localAddress, // the proxy socket is the actual outbound connection when tunneling HTTPS
                     method: 'CONNECT',
                     path,
                     headers: { Host: path, ...proxyAuth }

package/src/vfs.js

@@ -8,6 +8,7 @@ exports.saveVfs = exports.vfs = void 0;
 exports.permsFromParent = permsFromParent;
 exports.isSameFilenameAs = isSameFilenameAs;
 exports.normalizeFilename = normalizeFilename;
+exports.getFreeVfsName = getFreeVfsName;
 exports.applyParentToChild = applyParentToChild;
 exports.urlToNode = urlToNode;
 exports.nodeStats = nodeStats;
@@ -37,7 +38,12 @@ const comments_1 = require("./comments");
 const walkDir_1 = require("./walkDir");
 const node_stream_1 = require("node:stream");
 const adminApis_1 = require("./adminApis");
-const showHiddenFiles = (0, config_1.defineConfig)('show_hidden_files', false);
+const showHiddenFiles = (0, config_1.defineConfig)(misc_1.CFG.show_hidden_files, false);
+function setVfsPath(node, vfsPath, parent) {
+    return (0, misc_1.setHidden)(node, {
+        vfsPath: (0, misc_1.enforceFinal)('/', parent?.vfsPath) + (0, misc_1.pathEncode)(vfsPath)
+    });
+}
 function permsFromParent(parent, child) {
     const ret = {};
     for (const k of misc_1.PERM_KEYS) {
@@ -80,21 +86,29 @@ function isSameFilenameAs(name) {
 function normalizeFilename(x) {
     return (const_1.IS_WINDOWS || const_1.IS_MAC ? x.toLocaleLowerCase() : x).normalize();
 }
+function getFreeVfsName(siblings, name) {
+    const ext = (0, path_1.extname)(name);
+    const noExt = ext ? name.slice(0, -ext.length) : name;
+    let idx = 2;
+    while (siblings?.find(isSameFilenameAs(name)))
+        name = `${noExt} ${idx++}${ext}`;
+    return name;
+}
 async function applyParentToChild(child, parent, name) {
-    const ret = {
+    name ||= child ? getNodeName(child) : '';
+    const ret = setVfsPath({
         original: child, // this can be overridden by passing an 'original' in `child`
         ...child,
         isFolder: child?.isFolder ?? (child?.children?.length > 0 || undefined), // isFolder is hidden in original node, so we must copy it explicitly
         isTemp: true,
         parent,
-    };
-    name ||= child ? getNodeName(child) : '';
+    }, name, parent);
     inheritMasks(ret, parent, name);
     await parentMaskApplier(parent)(ret, name);
     inheritFromParent(ret);
     return ret;
 }
-async function urlToNode(url, ctx, parent = exports.vfs, allowMissing // true means missing path segments still resolve to temporary nodes with a computed source path
+async function urlToNode(url, ctx, parent = exports.vfs.compiled(), allowMissing // true means missing path segments still resolve to temporary nodes with a computed source path
 ) {
     let initialSlashes = 0;
     while (url[initialSlashes] === '/')
@@ -112,6 +126,7 @@ async function urlToNode(url, ctx, parent = exports.vfs, allowMissing // true me
     const ret = await getNodeByName(name, parent, assumeFolder);
     if (!ret)
         return;
+    setVfsPath(ret, name, parent);
     if (rest || ret?.original)
         return urlToNode(rest, ctx, ret, allowMissing);
     if (ret.source)
@@ -164,36 +179,49 @@ async function getNodeByName(name, parent, assumeMissingToBeFolder = false) {
         return ret;
     }
 }
-const smartUncFolderDetection = (0, config_1.defineConfig)('smart_unc_folder_detection', false);
+const smartUncFolderDetection = (0, config_1.defineConfig)(misc_1.CFG.smart_unc_folder_detection, false);
 async function setIsFolder(node) {
     if (!node.source)
         return;
-    const isFolder = /[\\/]$/.test(node.source)
+    const isFolder = (0, misc_1.hasFinalSlash)(node.source)
         || smartUncFolderDetection.get() && (0, misc_1.getUncHost)(node.source) && !(0, path_1.basename)(node.source).includes('.') // no dot = folder; not very reliable but fast for unreachable unc hosts, and it's an opt-in
         || await nodeStats(node).then(x => x?.isDirectory(), () => undefined);
     (0, misc_1.setHidden)(node, { isFolder });
+    if (isFolder)
+        persistFolderMarker(node);
     return isFolder;
 }
-exports.vfs = {};
-(0, config_1.defineConfig)('vfs', exports.vfs).sub(async (x) => {
-    await reviewVfs(x);
+// compiled is the stable mutable tree, while get() may return a fresh clone of the default
+exports.vfs = (0, config_1.defineConfig)(misc_1.CFG.vfs, {}, x => setVfsPath(structuredClone(x && typeof x === 'object' ? x : {}), '')); // ensure the type is right
+exports.vfs.sub(async () => {
+    await reviewVfs();
     console.log('VFS ready');
 });
-async function reviewVfs(data = exports.vfs) {
+async function reviewVfs() {
     await (async function recur(node) {
         if (node.source && !node.children?.length && node.isFolder === undefined)
             await setIsFolder(node);
-        if (node.children)
-            await Promise.allSettled(node.children.map(recur));
-    })(data);
-    exports.vfs = data;
+        if (!node.children)
+            return;
+        // we rename your node in case you got 2 nodes with the same name
+        const usedNames = new Set();
+        for (const child of node.children) {
+            const name = getNodeName(child);
+            const normalized = normalizeFilename(name);
+            if (usedNames.has(normalized))
+                child.name = getFreeVfsName(node.children, name);
+            usedNames.add(normalizeFilename(getNodeName(child)));
+        }
+        await Promise.allSettled(node.children.map(recur));
+    })(exports.vfs.compiled());
 }
 exports.saveVfs = (0, misc_1.debounceAsync)(async () => {
-    await reviewVfs();
-    await (0, config_1.setConfig)({ vfs: exports.vfs }, true);
+    await reviewVfs(); // refresh runtime-derived folder flags before saving mutated VFS state
+    exports.vfs.set(exports.vfs.compiled()); // sync the mutable runtime tree back to config state before persisting it
+    (0, config_1.saveConfigAsap)();
 });
 function isRoot(node) {
-    return node === exports.vfs;
+    return node === exports.vfs.compiled();
 }
 function getNodeName(node) {
     if (isRoot(node))
@@ -221,12 +249,24 @@ function nodeIsFolder(node) {
     function reconsider() {
         // a networked source may be offline at startup, and become online later: recalculate in the background
         nodeStats(node).then(s => {
-            if (s)
-                (0, misc_1.setHidden)(node.original || node, { isFolder: s.isDirectory() });
+            if (s) {
+                const isFolder = s.isDirectory();
+                (0, misc_1.setHidden)(node.original || node, { isFolder });
+                if (isFolder)
+                    persistFolderMarker(node);
+            }
         }, () => { });
         return undefined;
     }
 }
+// we mark folder paths with a final slash – the UI already does so, but the config may be modified
+function persistFolderMarker(node) {
+    const stored = node.original || node; // temporary VFS nodes must update their stored original for saveVfs to persist the marker
+    if (!stored.source || (0, misc_1.hasFinalSlash)(stored.source))
+        return;
+    stored.source += path_1.sep;
+    void (0, exports.saveVfs)();
+}
 async function getDefaultFile(node, ctx) {
     return node.default && nodeIsFolder(node) && await urlToNode(node.default, ctx, node) || undefined;
 }
@@ -266,12 +306,9 @@ function statusCodeForMissingPerm(node, perm, ctx, assign = true) {
         } while (1);
         if ((0, misc_1.isWhoObject)(who) || isWhoVfsPerms(who))
             throw Error(`permission type-guard: ${JSON.stringify(who)}`);
-        const eventName = 'checkVfsPermission';
-        if (events_1.default.anyListener(eventName)) {
-            const first = lodash_1.default.max(events_1.default.emit(eventName, { who, node, perm, ctx }));
-            if (first !== undefined)
-                return first;
-        }
+        const first = lodash_1.default.max(events_1.default.emit('checkVfsPermission', { who, node, perm, ctx }));
+        if (first !== undefined)
+            return first;
         return simpleWhoToError(who, ctx)
             ?? (0, misc_1.throw_)(Error(`invalid permission: ${perm}=${(0, misc_1.try_)(() => JSON.stringify(who))}`));
     }
@@ -306,10 +343,10 @@ async function* walkNode(parent, { ctx, depth = Infinity, prefixPath = '', requi
                 const nodeName = getNodeName(child);
                 const name = prefixPath + nodeName;
                 taken?.add(normalizeFilename(name));
-                const item = { ...child, original: child, name, parent };
+                const item = setVfsPath({ ...child, original: child, name, parent }, name, parent);
                 if (await cantSee(item))
                     return;
-                if (item.source && !item.children?.length) // real items must be accessible, unless there's more to it
+                if (item.source && !item.children?.length && !item.see_without_probing) // real items must be accessible, unless probing was explicitly disabled
                     try {
                         await promises_1.default.access(item.source);
                     }
@@ -349,7 +386,7 @@ async function* walkNode(parent, { ctx, depth = Infinity, prefixPath = '', requi
                         const name = prefixPath + (renamed || path);
                         if (taken?.has(normalizeFilename(name))) // taken by vfs node above
                             return false; // false just in case it's a folder
-                        const item = { name, isFolder, source: (0, path_1.join)(source, path), parent, stats: entry.stats };
+                        const item = setVfsPath({ name, isFolder, source: (0, path_1.join)(source, path), parent, stats: entry.stats }, name, parent);
                         // masks containing '/' must be matched against the relative path while keeping walkDir recursion enabled
                         await pathMaskApplier(item, renamed || path);
                         if (await cantSee(item)) // can't see: don't produce and don't recur
@@ -482,7 +519,7 @@ events_1.default.on('accountRenamed', ({ from, to }) => {
         if (n.masks)
             Object.values(n.masks).forEach(renameInNode);
         n.children?.forEach(renameInNode);
-    })(exports.vfs);
+    })(exports.vfs.compiled());
     (0, exports.saveVfs)();
     function renameInPerm(a) {
         if ((0, misc_1.isWhoObject)(a)) {

package/src/walkDir.js

@@ -23,12 +23,12 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
     return new Promise(async (resolve, reject) => {
         if (!await (0, util_files_1.isDirectory)(path))
             return reject(Error('ENOTDIR'));
-        dirQ.add(() => readDir('', depth)
+        dirQ.add(() => readDir('', depth, [path])
             .then(res => {
             Promise.resolve(res?.branchDone).then(resolve);
         }, reject));
     });
-    async function readDir(relativePath, depth) {
+    async function readDir(relativePath, depth, ancestorPaths) {
         if (stopped)
             return;
         const base = (0, path_1.join)(path, relativePath);
@@ -39,6 +39,8 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
         const pluginReceiver = lodash_1.default.isFunction(res) && res || null;
         const pluginIterator = lodash_1.default.isFunction(res?.[Symbol.asyncIterator] || res?.[Symbol.iterator]) && res;
         if (const_1.IS_WINDOWS && !pluginIterator) { // use native apis to read the 'hidden' attribute
+            // fswin callbacks cannot await, so track their work before closing the branch
+            const entriesWorking = [];
             const direntMethods = {
                 isDir: false,
                 isFile() { return !this.isDir; },
@@ -46,14 +48,16 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
                 isBlockDevice() { return false; },
                 isCharacterDevice() { return false; },
             };
-            await new Promise(res => fswin_1.default.find(base + '\\*', (event, f) => {
-                if (event !== 'FOUND')
-                    return res();
+            await new Promise((resolve, reject) => fswin_1.default.find(base + '\\*', (event, f) => {
+                if (event !== 'FOUND') {
+                    Promise.all(entriesWorking).then(() => resolve());
+                    return;
+                }
                 if (stopped)
                     return true; // stop signal
                 if (!hidden && f.IS_HIDDEN)
                     return;
-                work(Object.assign(Object.create(direntMethods), {
+                entriesWorking.push(work(Object.assign(Object.create(direntMethods), {
                     isDir: f.IS_DIRECTORY,
                     name: f.LONG_NAME,
                     stats: {
@@ -63,7 +67,7 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
                         isFile: () => !f.IS_DIRECTORY,
                         isDirectory: () => f.IS_DIRECTORY,
                     }
-                }));
+                }), Boolean(f.REPARSE_POINT_TAG)).catch(reject));
             }, true));
         }
         else
@@ -72,7 +76,9 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
                     break;
                 if (!hidden && !const_1.IS_WINDOWS && entry.name[0] === '.')
                     continue;
-                const stats = entry.isSymbolicLink?.() && await (0, util_files_1.statWithTimeout)((0, path_1.join)(base, entry.name)).catch(() => null);
+                // stat follows links, so preserve their identity for cycle detection
+                const isSymlink = entry.isSymbolicLink?.();
+                const stats = isSymlink && await (0, util_files_1.statWithTimeout)((0, path_1.join)(base, entry.name)).catch(() => null);
                 if (stats === null)
                     continue;
                 if (stats)
@@ -80,7 +86,7 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
                 const expanded = entry;
                 if (stats)
                     expanded.stats = stats;
-                await work(expanded);
+                await work(expanded, isSymlink);
             }
         pluginReceiver?.(!stopped);
         const branchDone = Promise.allSettled(subDirsDone);
@@ -90,7 +96,7 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
             closingQ.push(relativePath); // ok, we'll ask next one to carry this info
         // don't return the promise directly, as this job ends here, but communicate to caller the promise for the whole branch
         return { branchDone, n };
-        async function work(entry) {
+        async function work(entry, isSymlink = false) {
             entry.path = (relativePath && relativePath + '/') + entry.name;
             pluginReceiver?.(entry);
             if (last && closingQ.length) // pending entries
@@ -104,9 +110,16 @@ function walkDir(path, { depth = 0, hidden = true, parallelizeRecursion = false,
             n++;
             if (!depth || !entry.isDirectory())
                 return;
+            const entryPath = (0, path_1.join)(base, entry.name);
+            if (isSymlink) {
+                // compare only with ancestors so separate links to the same folder still work
+                const [target, ...ancestors] = await Promise.all([entryPath, ...ancestorPaths].map(x => (0, promises_1.realpath)(x).catch(() => '')));
+                if (!target || ancestors.includes(target))
+                    return;
+            }
             const branchDone = (0, cross_1.pendingPromise)(); // per-job
             subDirsDone.push(branchDone);
-            const job = () => readDir(entry.path, depth - 1) // recur
+            const job = () => readDir(entry.path, depth - 1, [...ancestorPaths, entryPath]) // recur
                 .then(x => x, () => { }) // mute errors
                 .then(res => {
                 if (!res?.n)

package/src/watchLoad.js

@@ -12,8 +12,9 @@ const debounceAsync_1 = require("./debounceAsync");
 const events_1 = require("./events");
 function watchLoad(path, parser, { failedOnFirstAttempt, immediateFirst } = {}) {
     let doing = false;
+    let cancelled = false;
     let watcher;
-    const debounced = (0, debounceAsync_1.debounceAsync)(load, { wait: 500, maxWait: 1000, reuseRunning: true });
+    const debounced = (0, debounceAsync_1.debounceAsync)(load, { wait: 500, maxWait: 1000, reuseRunning: true, cancelable: true });
     let retry;
     let last;
     const emitter = new events_1.BetterEventEmitter();
@@ -27,6 +28,8 @@ function watchLoad(path, parser, { failedOnFirstAttempt, immediateFirst } = {})
     install(true);
     return { unwatch, save, emitter, getText: () => last, getPath: () => path };
     function install(first = false) {
+        if (cancelled)
+            return;
         try {
             watcher = (0, fs_1.watch)(path, () => {
                 if (!save.isWorking())
@@ -43,6 +46,12 @@ function watchLoad(path, parser, { failedOnFirstAttempt, immediateFirst } = {})
         }
     }
     function unwatch() {
+        // a pending or running load must not resurrect a watcher cancelled by its owner
+        cancelled = true;
+        debounced.cancel();
+        closeWatcher();
+    }
+    function closeWatcher() {
         watcher?.close();
         clearTimeout(retry);
         watcher = undefined;
@@ -61,12 +70,14 @@ function watchLoad(path, parser, { failedOnFirstAttempt, immediateFirst } = {})
                     return '';
                 throw e;
             });
+            if (cancelled)
+                return;
             if (text === last)
                 return;
             last = text;
             emitter.emit('change', last);
             console.debug('Loaded', path);
-            unwatch();
+            closeWatcher();
             install(); // reinstall, as the original file could have been renamed. We watch by the name.
             await parser(text);
         }