@e-mc/file-manager 0.14.0 → 0.14.2

package/README.md

@@ -9,7 +9,7 @@
 
 ## Interface
 
-* [View Source](https://www.unpkg.com/@e-mc/types@0.14.0/lib/index.d.ts)
+* [View Source](https://www.unpkg.com/@e-mc/types@0.14.2/lib/index.d.ts)
 
 ```typescript
 import type { ChecksumValue, DataSource, IncrementalMatch, TaskAction } from "./squared";
@@ -297,15 +297,15 @@ NOTE: **FileManager** is a sub-class of [Host](https://www.npmjs.com/package/@e-
 
 ## References
 
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/squared.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/asset.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/core.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/filemanager.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/logger.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/module.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/node.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/request.d.ts
-- https://www.unpkg.com/@e-mc/types@0.14.0/lib/settings.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/squared.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/asset.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/core.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/filemanager.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/logger.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/module.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/node.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/request.d.ts
+- https://www.unpkg.com/@e-mc/types@0.14.2/lib/settings.d.ts
 
 * https://www.npmjs.com/package/@types/node
 

package/index.js

@@ -12,7 +12,7 @@ const { DOWNLOAD_TYPE, FILE_TYPE, THRESHOLD, asExt, cloneFlag, convertTime, crea
 const { AbortComponent, Host, Permission } = require('@e-mc/core');
 const { isEqual, setInitialValue } = require('@e-mc/document/asset');
 const { concatString, getNewline } = require('@e-mc/document/util');
-const { asFloat, asInt, byteLength, checkRetryable, fromSeconds, fromURL, getSize, getTransferRate, hasSameStat, hasSize } = require('@e-mc/request/util');
+const { asFloat, asInt, byteLength, checkRetryable, fromSeconds, fromURL, getSize, getTransferRate, hasSameStat, hasSize, parseHeader } = require('@e-mc/request/util');
 const Request = require('@e-mc/request');
 const Image = require('@e-mc/image');
 const Watch = require('@e-mc/watch');
@@ -134,7 +134,7 @@ function recurseDir(output, subDirs, options) {
     const baseDir = path.join(...subDirs);
     const items = fs.readdirSync(baseDir, { withFileTypes: true })
         .filter(item => {
-        if (ignoreGlob && ignoreGlob(fromDir ? path.posix.join(fromDir, item.name) : item.name)) {
+        if (ignoreGlob?.(fromDir ? path.posix.join(fromDir, item.name) : item.name)) {
             return false;
         }
         return item.name;
@@ -263,7 +263,7 @@ function equalAddress(value, other) {
     const b = decodeURIComponent(other);
     return a === b || a === other || value === b;
 }
-function processHeaders(item, headers, lastModified, mainEtag) {
+function processHeaders(headers, item, lastModified, mainEtag) {
     const contentLength = parseInt(headers['content-length']);
     if (contentLength > 0) {
         item.contentLength = contentLength;
@@ -282,6 +282,36 @@ function processHeaders(item, headers, lastModified, mainEtag) {
         return item.lastModified = headers['last-modified'];
     }
 }
+function processCacheHeaders(headers, noCache, noStore) {
+    if (noStore) {
+        return NaN;
+    }
+    const cacheControl = parseHeader(headers, 'cache-control');
+    if (cacheControl) {
+        if (cacheControl.includes('no-store')) {
+            return -1;
+        }
+        const maxAge = cacheControl.find(value => value.startsWith('max-age='));
+        if (maxAge && (!noCache || cacheControl.includes('public') || cacheControl.includes('must-revalidate'))) {
+            const age = parseHeader(headers, 'age');
+            let result = parseInt(maxAge.split('=').pop()) * 1000;
+            if (age) {
+                result -= age;
+                if (result < 0 && noCache) {
+                    return -1;
+                }
+            }
+            return result > 0 ? result : 0;
+        }
+    }
+    if (!noCache) {
+        const expires = parseHeader(headers, 'expires');
+        if (expires && !isNaN(expires)) {
+            const date = parseHeader(headers, 'date') || Date.now();
+            return expires - date;
+        }
+    }
+}
 function setContent(item, buffer) {
     if (typeof buffer === 'string') {
         item.sourceUTF8 = buffer;
@@ -312,7 +342,7 @@ async function doVerifyChecksum(root, from, options, nested) {
         const filename = path.basename(from);
         const files = [];
         from = joinRoot ? path.join(root, filename) : path.resolve(from);
-        recurseDir(files, [root], { ignore: mapPosix([...ignore, from]), ignoreRoot: mapPosix(ignoreRoot), sortBy, recursive });
+        recurseDir(files, [root], { ignore: toPosix([...ignore, from]), ignoreRoot: toPosix(ignoreRoot), sortBy, recursive });
         const items = fs.readFileSync(from, 'utf8').split('\n').map(item => {
             const index = item.indexOf(' ');
             return [item.slice(0, index), path.join(root, item.slice(index + 1))];
@@ -468,13 +498,15 @@ const downloadStats = () => [[0, 0], [0, 0], [0, 0]];
 const isCacheable = (item) => item.initialValue?.cacheable !== false;
 const getPmOptions = (value, { matchBase = value.startsWith('*') && !value.includes('/'), dot = false } = {}) => ({ nocase: Host.PLATFORM_WIN32, matchBase, dot });
 const isDownloadAll = (type) => type === 4 || type === 0;
+const isNoContent = (statusCode) => statusCode === 204 || statusCode === 304;
+const isStored = (noCache, maxAge) => !noCache || typeof maxAge === 'number';
 const hasFiles = (type, uri) => type === 3 || type === 4 || type === 0 && Request.isRclone(uri);
 const hasIncremental = (value) => value === "etag" || value === "exists";
-const matchPathname = (value) => Host.PLATFORM_WIN32 ? value.toLowerCase() : value;
-const mapPosix = (values) => Host.PLATFORM_WIN32 ? values.map(value => Permission.toPosix(value).toLowerCase()) : values;
+const toPathCase = (value) => Host.PLATFORM_WIN32 ? value.toLowerCase() : value;
+const toPosix = (values) => Host.PLATFORM_WIN32 ? values.map(value => Permission.toPosix(value).toLowerCase()) : values;
 const formatPercent = (value) => Math.round(value).toString().padStart(3) + '%';
 const formatLength = (value, length) => `${length} ${value + (length === 1 ? '' : 's')}`;
-const checkEOF = (value) => value.endsWith('\n') ? value : value + '\n';
+const checkEOF = (value) => value.at(-1) === '\n' ? value : value + '\n';
 const checksumFile = (algorithm) => "checksum" + '.' + (isString(algorithm) ? algorithm.toLowerCase() : "sha256");
 const ignoreAsset = (item, exists) => item.invalid || hasBit(item.flags, 1 | (!exists ? 128 : 0));
 const validateChecksum = (host, item, buffer, localUri = item.localUri) => !item.checksum || host.checkHash(item.checksum, buffer, localUri);
@@ -609,13 +641,13 @@ class HttpDiskCache {
         if (!this.enabled) {
             return false;
         }
-        try {
-            if (typeof uri === 'string') {
+        if (typeof uri === 'string') {
+            try {
                 uri = new URL(uri);
             }
-        }
-        catch {
-            return false;
+            catch {
+                return false;
+            }
         }
         const flags = this._flags;
         if (flags === 0) {
@@ -639,7 +671,14 @@ class HttpDiskCache {
         }
         return true;
     }
-    add(uri, etag, target, { buffer, contentLength = getSize(target) } = {}) {
+    add(uri, etag, target, { buffer, contentLength = getSize(target), maxAge = 0 } = {}) {
+        if (maxAge < 0) {
+            this.clear(target);
+            return;
+        }
+        if (isNaN(maxAge)) {
+            return;
+        }
         let tempDir;
         if (contentLength <= this.limit && (tempDir = this.host.getCacheDir(uri))) {
             const baseDir = path.join(tempDir, etag);
@@ -654,7 +693,7 @@ class HttpDiskCache {
                 else {
                     fs.copyFileSync(target, tempUri);
                 }
-                const expires = this.expires;
+                const expires = maxAge || this.expires;
                 if (expires > 0 && expires < Infinity) {
                     setTimeout(() => {
                         this.clear(target);
@@ -741,7 +780,14 @@ class HttpMemoryCache extends HttpDiskCache {
     has(uri) {
         return this.expires > 0 && super.has(uri);
     }
-    add(uri, etag, buffer, { encoding, toDisk, contentLength = Buffer.byteLength(buffer, encoding) } = {}) {
+    add(uri, etag, buffer, { encoding, toDisk, contentLength = Buffer.byteLength(buffer, encoding), maxAge = 0 } = {}) {
+        if (maxAge < 0) {
+            this.clear(uri);
+            return;
+        }
+        if (isNaN(maxAge)) {
+            return;
+        }
         if (contentLength <= this.limit) {
             if (uri instanceof URL) {
                 uri = uri.href;
@@ -753,10 +799,11 @@ class HttpMemoryCache extends HttpDiskCache {
             const item = [Date.now(), buffer, encoding, contentLength, uri, etag];
             this._items.push(item);
             MEMORY.CACHE.set(uri, item);
-            if (this.expires < Infinity) {
+            const expires = maxAge || this.expires;
+            if (expires < Infinity) {
                 setTimeout(() => {
                     this.clear(uri);
-                }, Math.min(this.expires, Host.MAX_TIMEOUT));
+                }, Math.min(expires, Host.MAX_TIMEOUT));
             }
         }
         else {
@@ -862,10 +909,14 @@ class ProcessFile {
                     const parent = host.assets.find(item => item.bundleIndex === 0 && item.bundleId === bundleId);
                     if (parent) {
                         (parent.bundleQueue ||= []).push(new Promise(resolve => {
-                            host.Request.open(url, { method: 'HEAD', httpVersion: 1 })
+                            host.Request.open(url, { method: 'HEAD', httpVersion: 1, headers: file.headers })
                                 .on('response', res => {
-                                if (res.statusCode < 300) {
-                                    processHeaders(file, res.headers, false, false);
+                                const statusCode = res.statusCode;
+                                if (isNoContent(statusCode)) {
+                                    file.contentLength = 0;
+                                }
+                                else if (statusCode < 300) {
+                                    processHeaders(res.headers, file, false, false);
                                 }
                                 resolve(file);
                             })
@@ -992,11 +1043,11 @@ class ProcessFile {
                     for (const queue of items) {
                         const encoding = queue.encoding ||= 'utf8';
                         const { uri, fetchType: type } = queue;
-                        let tempFile;
+                        let tempFile, maxAge;
                         if (checkEtag && queue.trailingContent) {
                             checkEtag = false;
                         }
-                        const verifyBundle = (value, etag, checked) => {
+                        const verifyBundle = (value, etag, checked, noCache) => {
                             if (!queue.invalid) {
                                 if (!checked && !validateChecksum(host, queue, value, localUri)) {
                                     queue.invalid = true;
@@ -1006,10 +1057,11 @@ class ProcessFile {
                                     value = value.toString(encoding);
                                 }
                                 const url = queue.url;
-                                if (etag && host.cacheToMemory.has(url)) {
+                                if (etag && host.cacheToMemory.has(url) && isStored(noCache, maxAge)) {
                                     host.cacheToMemory.add(url, encodeURIComponent(etag), value, {
-                                        encoding,
                                         contentLength: queue.contentLength,
+                                        maxAge,
+                                        encoding,
                                         toDisk: !tempFile && isCacheable(queue) ? queue.filename || queue.localUri && path.basename(queue.localUri) : ''
                                     });
                                 }
@@ -1026,6 +1078,8 @@ class ProcessFile {
                                 const options = {
                                     url,
                                     encoding,
+                                    headers: queue.headers,
+                                    noCache: true,
                                     statusMessage: uri + ` (${queue.bundleIndex})`
                                 };
                                 let etag, pipeTo;
@@ -1078,14 +1132,15 @@ class ProcessFile {
                                         options.httpVersion = 1;
                                     }
                                     options.connected = (headers) => {
-                                        etag = processHeaders(queue, headers, !!host.Watch, false);
+                                        etag = processHeaders(headers, queue, !!host.Watch, false);
+                                        maxAge = processCacheHeaders(headers, options.noCache, options.noStore);
                                         return true;
                                     };
                                 }
                                 tasks.push(new Promise((resolve, reject) => {
                                     void host.scheduleTask(url || uri, options, (data) => {
                                         if (data) {
-                                            verifyBundle(data, etag);
+                                            verifyBundle(data, etag, false, options.noCache);
                                         }
                                         else {
                                             queue.invalid = true;
@@ -1093,12 +1148,7 @@ class ProcessFile {
                                         const downloaded = groupData.bundling[uri];
                                         if (isArray(downloaded)) {
                                             if (data && !queue.invalid) {
-                                                if (typeof data === 'string') {
-                                                    queue.sourceUTF8 = data;
-                                                }
-                                                else {
-                                                    queue.buffer = data;
-                                                }
+                                                setContent(queue, data);
                                                 this.copyDownload(queue, downloaded);
                                             }
                                             else {
@@ -1129,7 +1179,7 @@ class ProcessFile {
                                 }
                                 const pathname = getTempDir(true);
                                 tasks.push(new Promise((resolve, reject) => {
-                                    void host.scheduleTask(queue.url || uri, { pathname, binOpts: queue.binOpts || file.binOpts }, (result) => {
+                                    void host.scheduleTask(queue.url || uri, { pathname, binOpts: queue.binOpts || file.binOpts, headers: queue.headers }, (result) => {
                                         if (result.length > 0) {
                                             verifyBundle(bundleTorrent(host, result, mimeType, encoding));
                                         }
@@ -1286,6 +1336,7 @@ class FileManager extends Host {
                 result = MEMORY.CACHE.size;
                 MEMORY.CACHE.clear();
                 MEMORY.SIZE = 0;
+                CACHE_ETAG.clear();
             }
             else if (percent > 0) {
                 const items = Array.from(MEMORY.CACHE.entries());
@@ -1463,7 +1514,7 @@ class FileManager extends Host {
         try {
             const filename = path.basename(to);
             to = joinRoot ? path.join(root, to) : path.resolve(to);
-            recurseDir(result, [root], { ignore: mapPosix([...ignore, to]), ignoreRoot: mapPosix(ignoreRoot), sortBy, recursive });
+            recurseDir(result, [root], { ignore: toPosix([...ignore, to]), ignoreRoot: toPosix(ignoreRoot), sortBy, recursive });
             const output = [];
             for (const pathname of result = filterPaths(root, result, options)) {
                 if (recursive === 1 && path.basename(pathname) === filename) {
@@ -1604,7 +1655,7 @@ class FileManager extends Host {
         if (LOG_TIMEPROCESS) {
             Host.initCpuUsage(this);
         }
-        this.#baseDirectory = path.normalize(baseDirectory.at(-1) === path.sep ? baseDirectory.slice(0, -1) : baseDirectory);
+        this.#baseDirectory = path.normalize(Host.PLATFORM_WIN32 ? baseDirectory.replace(/[\\/]+$/, '') : baseDirectory.replace(/\/+$/, ''));
         this.permission = permission && Host.isPermission(permission) ? permission : Host.getPermissionFromSettings();
         this.sessionId = FileManager.generateSessionId();
         this.supports('permission', false, true);
@@ -2331,13 +2382,13 @@ class FileManager extends Host {
                 pathname = file.pathname || '';
             }
             const assets = this.assets;
-            let target = matchPathname(path.join(pathname, filename)), modified = false;
+            let target = toPathCase(path.join(pathname, filename)), modified = false;
             for (let i = 0, j = 1, length = assets.length; i < length; ++i) {
                 const item = assets[i];
-                if (item !== file && item.filename && target === matchPathname(path.join(item.pathname || '', item.filename))) {
+                if (item !== file && item.filename && target === toPathCase(path.join(item.pathname || '', item.filename))) {
                     const { name, ext } = path.parse(filename);
                     filename = name + '_' + j + ext;
-                    target = matchPathname(path.join(pathname, filename));
+                    target = toPathCase(path.join(pathname, filename));
                     modified = true;
                     i = -1;
                     ++j;
@@ -3464,14 +3515,17 @@ class FileManager extends Host {
                             }
                             client?.destroy();
                             const location = request.url.toString();
+                            let maxAge;
                             request.encoding = encoding;
+                            request.headers = item.headers;
                             request.pipeTo = localUri;
                             request.statusMessage = location + (bundleMain ? ' (0)' : '');
                             if (type === 1) {
                                 request.method = 'GET';
                                 request.httpVersion = (originCount[request.url.origin] || 0) <= 1 ? 1 : undefined;
                                 request.connected = headers => {
-                                    processHeaders(item, headers, watching, mainEtag);
+                                    processHeaders(headers, item, watching, mainEtag);
+                                    maxAge = processCacheHeaders(headers, request.noCache, request.noStore);
                                     return item.willChange || !!etagDir && (cacheDir || !item.contentLength || cacheToMemory.within(item.contentLength)) || this.hasLog('progress');
                                 };
                             }
@@ -3488,13 +3542,13 @@ class FileManager extends Host {
                                     item.buffer = data;
                                 }
                                 target.received(null, incremental, type === 1);
-                                if (etagDir) {
+                                if (etagDir && isStored(request.noCache, maxAge)) {
                                     queueMicrotask(() => {
                                         if (cacheBuffer && data) {
-                                            cacheToMemory.add(location, etagDir, data, { encoding, contentLength: item.contentLength, toDisk: !cacheDir && isCacheable(item) ? path.basename(localUri) : '' });
+                                            cacheToMemory.add(location, etagDir, data, { contentLength: item.contentLength, maxAge, encoding, toDisk: !cacheDir && isCacheable(item) ? path.basename(localUri) : '' });
                                         }
                                         if (cacheDir) {
-                                            cacheToDisk.add(location, etagDir, localUri, { buffer: data, contentLength: item.contentLength });
+                                            cacheToDisk.add(location, etagDir, localUri, { contentLength: item.contentLength, maxAge, buffer: data });
                                         }
                                     });
                                 }
@@ -3508,15 +3562,18 @@ class FileManager extends Host {
                                 const request = this.Request;
                                 try {
                                     ++processTask.pending;
-                                    const opts = request.opts(href, { method: 'HEAD', httpVersion: 1 });
+                                    const opts = request.opts(href, { method: 'HEAD', httpVersion: 1, headers: item.headers });
                                     (client = request.open(href, opts))
                                         .on('response', res => {
                                         --processTask.pending;
                                         const statusCode = res.statusCode;
-                                        if (statusCode < 300) {
-                                            const etag = processHeaders(item, res.headers, false, mainEtag);
+                                        if (isNoContent(statusCode)) {
+                                            downloadUri(opts, null);
+                                        }
+                                        else if (statusCode < 300) {
+                                            const etag = processHeaders(res.headers, item, false, mainEtag && !opts.noStore);
                                             let etagDir = null, tempDir;
-                                            if (etag) {
+                                            if (etag && !opts.noCache && !opts.noStore) {
                                                 if (cacheDir || mainEtag) {
                                                     tempDir = this.getCacheDir(href);
                                                 }
@@ -3557,7 +3614,7 @@ class FileManager extends Host {
                                                                         return;
                                                                     }
                                                                     if (cacheToMemory.within(item.contentLength = Buffer.byteLength(buffer, encoding))) {
-                                                                        cacheToMemory.add(location, etagDir, buffer, { encoding, contentLength: item.contentLength, tempFile: pipeAs });
+                                                                        cacheToMemory.add(location, etagDir, buffer, { contentLength: item.contentLength, encoding, tempFile: pipeAs });
                                                                     }
                                                                 }
                                                                 if (checkDest(pipeAs, localUri)) {
@@ -3670,7 +3727,7 @@ class FileManager extends Host {
                     }
                     downloading[uri] = [];
                     this.performAsyncTask();
-                    const options = { pathname, binOpts: item.binOpts, shellExpansion: item.shellExpansion };
+                    const options = { pathname, binOpts: item.binOpts, shellExpansion: item.shellExpansion, headers: item.headers };
                     let src;
                     if (type === 0) {
                         if (item.filename) {
@@ -4289,12 +4346,12 @@ class FileManager extends Host {
         });
         instance.on('dir:remove', value => {
             if (this.removeCwd(value)) {
-                value = matchPathname(value);
+                value = toPathCase(value);
                 if (!value.endsWith(path.sep)) {
                     value += path.sep;
                 }
                 for (const name of this.files) {
-                    if (matchPathname(name).startsWith(value)) {
+                    if (toPathCase(name).startsWith(value)) {
                         this.files.delete(name);
                     }
                 }

package/package.json

@@ -1,12 +1,9 @@
 {
   "name": "@e-mc/file-manager",
-  "version": "0.14.0",
+  "version": "0.14.2",
   "description": "FileManager constructor for E-mc.",
   "main": "index.js",
   "types": "index.d.ts",
-  "publishConfig": {
-    "access": "public"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/anpham6/e-mc.git",
@@ -19,15 +16,15 @@
   "license": "BSD-3-Clause",
   "homepage": "https://github.com/anpham6/e-mc#readme",
   "dependencies": {
-    "@e-mc/cloud": "0.14.0",
-    "@e-mc/compress": "0.14.0",
-    "@e-mc/core": "0.14.0",
-    "@e-mc/document": "0.14.0",
-    "@e-mc/image": "0.14.0",
-    "@e-mc/request": "0.14.0",
-    "@e-mc/task": "0.14.0",
-    "@e-mc/types": "0.14.0",
-    "@e-mc/watch": "0.14.0",
+    "@e-mc/cloud": "0.14.2",
+    "@e-mc/compress": "0.14.2",
+    "@e-mc/core": "0.14.2",
+    "@e-mc/document": "0.14.2",
+    "@e-mc/image": "0.14.2",
+    "@e-mc/request": "0.14.2",
+    "@e-mc/task": "0.14.2",
+    "@e-mc/types": "0.14.2",
+    "@e-mc/watch": "0.14.2",
     "chalk": "4.1.2",
     "diff": "^9.0.0",
     "picomatch": "^4.0.4"