@filen/sync 0.1.1 → 0.1.3

package/dist/lib/filesystems/remote.js

@@ -4,43 +4,278 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RemoteFileSystem = void 0;
+const sdk_1 = require("@filen/sdk");
 const path_1 = __importDefault(require("path"));
 const semaphore_1 = require("../../semaphore");
 const fs_extra_1 = __importDefault(require("fs-extra"));
+const ipc_1 = require("../ipc");
+const utils_1 = require("../../utils");
+const uuid_1 = require("uuid");
+const constants_1 = require("../../constants");
 class RemoteFileSystem {
-    constructor({ sync }) {
+    constructor(sync) {
         this.getDirectoryTreeCache = {
             timestamp: 0,
             tree: {},
             uuids: {}
         };
+        this.previousTreeRawResponse = "";
         this.mutex = new semaphore_1.Semaphore(1);
         this.mkdirMutex = new semaphore_1.Semaphore(1);
+        this.itemsMutex = new semaphore_1.Semaphore(1);
+        this.deviceIdCache = "";
         this.sync = sync;
     }
-    async getDirectoryTree() {
-        // TODO: Actual implementation using cache + different endpoint with deviceId
-        // Account for duplicate path names for directories/files. Only keep the first one discovered in the tree.
+    async getDeviceId() {
+        if (this.deviceIdCache.length > 0) {
+            return this.deviceIdCache;
+        }
+        const deviceIdFile = path_1.default.join(this.sync.dbPath, this.sync.syncPair.uuid, "deviceId");
+        await fs_extra_1.default.ensureDir(path_1.default.dirname(deviceIdFile));
+        let deviceId = "";
+        if (!(await fs_extra_1.default.exists(deviceIdFile))) {
+            deviceId = (0, uuid_1.v4)();
+            await fs_extra_1.default.writeFile(deviceIdFile, deviceId, {
+                encoding: "utf-8"
+            });
+        }
+        else {
+            deviceId = await fs_extra_1.default.readFile(deviceIdFile, {
+                encoding: "utf-8"
+            });
+        }
+        this.deviceIdCache = deviceId;
+        return deviceId;
+    }
+    async getDirectoryTree(skipCache = false) {
+        var _a;
+        const deviceId = await this.getDeviceId();
+        const dir = await this.sync.sdk.api(3).dir().tree({
+            uuid: this.sync.syncPair.remoteParentUUID,
+            deviceId,
+            skipCache,
+            includeRaw: true
+        });
+        // Data did not change, use cache
+        if (dir.files.length === 0 && dir.folders.length === 0) {
+            if (this.getDirectoryTreeCache.timestamp === 0) {
+                // Re-run but with API caching disabled since our internal client cache is empty
+                return await this.getDirectoryTree(true);
+            }
+            return {
+                result: this.getDirectoryTreeCache,
+                ignored: [],
+                changed: false
+            };
+        }
+        // eslint-disable-next-line quotes
+        const rawEx = dir.raw.split('"randomBytes"');
+        // Compare API response with the previous dataset, this way we can save time computing the tree if it's the same
+        if (rawEx.length === 2 && this.previousTreeRawResponse === rawEx[0] && this.getDirectoryTreeCache.timestamp !== 0) {
+            return {
+                result: this.getDirectoryTreeCache,
+                ignored: [],
+                changed: false
+            };
+        }
+        const baseFolder = dir.folders[0];
+        if (!baseFolder) {
+            throw new Error("Could not get base folder.");
+        }
+        const baseFolderParent = baseFolder[2];
+        if (baseFolderParent !== "base") {
+            throw new Error("Invalid base folder parent.");
+        }
+        const folderNames = { base: "/" };
         const tree = {};
-        const dir = await this.sync.sdk.cloud().getDirectoryTree({ uuid: this.sync.syncPair.remoteParentUUID });
         const uuids = {};
-        for (const path in dir) {
-            if (!dir[path] || dir[path].parent === "base" || path.startsWith(".filen.trash.local")) {
-                continue;
+        const pathsAdded = {};
+        const ignored = [];
+        for (const folder of dir.folders) {
+            try {
+                const decrypted = await this.sync.sdk.crypto().decrypt().folderMetadata({ metadata: folder[1] });
+                if (folder[2] !== "base" && decrypted.name.length === 0) {
+                    continue;
+                }
+                const parentPath = folder[2] === "base" ? "" : `${folderNames[folder[2]]}/`;
+                const folderPath = folder[2] === "base" ? "" : `${parentPath}${decrypted.name}`;
+                const localPath = path_1.default.join(this.sync.syncPair.localPath, folderPath);
+                folderNames[folder[0]] = folderPath;
+                if (folderPath.startsWith(constants_1.LOCAL_TRASH_NAME) || decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME)) {
+                    continue;
+                }
+                if (this.sync.remoteIgnorer.ignores(folderPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "remoteIgnore"
+                    });
+                    continue;
+                }
+                if (this.sync.excludeDotFiles && decrypted.name.startsWith(".")) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "dotFile"
+                    });
+                    continue;
+                }
+                if ((0, utils_1.isPathOverMaxLength)(localPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "pathLength"
+                    });
+                    continue;
+                }
+                if ((0, utils_1.isNameOverMaxLength)(decrypted.name)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "nameLength"
+                    });
+                    continue;
+                }
+                if (!(0, utils_1.isValidPath)(localPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "invalidPath"
+                    });
+                    continue;
+                }
+                if ((0, utils_1.isDirectoryPathIgnoredByDefault)(folderPath) || (0, utils_1.isRelativePathIgnoredByDefault)(folderPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: folderPath,
+                        reason: "defaultIgnore"
+                    });
+                    continue;
+                }
+                const lowercasePath = folderPath.toLowerCase();
+                if (pathsAdded[lowercasePath]) {
+                    continue;
+                }
+                pathsAdded[lowercasePath] = true;
+                const item = {
+                    type: "directory",
+                    uuid: folder[0],
+                    name: decrypted.name,
+                    size: 0,
+                    path: folderPath
+                };
+                tree[folderPath] = item;
+                uuids[folder[0]] = item;
             }
-            const item = Object.assign(Object.assign({}, dir[path]), { path });
-            tree[path] = item;
-            const treeItem = tree[path];
-            if (treeItem) {
-                uuids[treeItem.uuid] = item;
+            catch (e) {
+                this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
             }
         }
+        if (Object.keys(folderNames).length === 0) {
+            throw new Error("Could not build directory tree.");
+        }
+        await (0, utils_1.promiseAllSettledChunked)(dir.files.map(async (file) => {
+            try {
+                const decrypted = await this.sync.sdk.crypto().decrypt().fileMetadata({ metadata: file[5] });
+                if (decrypted.name.length === 0) {
+                    return;
+                }
+                const parentPath = folderNames[file[4]];
+                const filePath = `${parentPath}/${decrypted.name}`;
+                const localPath = path_1.default.join(this.sync.syncPair.localPath, filePath);
+                if (filePath.startsWith(constants_1.LOCAL_TRASH_NAME) || decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME)) {
+                    return;
+                }
+                if (this.sync.remoteIgnorer.ignores(filePath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "remoteIgnore"
+                    });
+                    return;
+                }
+                if (this.sync.excludeDotFiles && decrypted.name.startsWith(".")) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "dotFile"
+                    });
+                    return;
+                }
+                if ((0, utils_1.isPathOverMaxLength)(localPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "pathLength"
+                    });
+                    return;
+                }
+                if ((0, utils_1.isNameOverMaxLength)(decrypted.name)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "nameLength"
+                    });
+                    return;
+                }
+                if (!(0, utils_1.isValidPath)(localPath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "invalidPath"
+                    });
+                    return;
+                }
+                if ((0, utils_1.isDirectoryPathIgnoredByDefault)(filePath) || (0, utils_1.isRelativePathIgnoredByDefault)(filePath)) {
+                    ignored.push({
+                        localPath,
+                        relativePath: filePath,
+                        reason: "defaultIgnore"
+                    });
+                    return;
+                }
+                const lowercasePath = filePath.toLowerCase();
+                if (pathsAdded[lowercasePath]) {
+                    return;
+                }
+                pathsAdded[lowercasePath] = true;
+                const item = {
+                    type: "file",
+                    uuid: file[0],
+                    name: decrypted.name,
+                    size: decrypted.size,
+                    mime: decrypted.mime,
+                    lastModified: (0, utils_1.convertTimestampToMs)(decrypted.lastModified),
+                    version: file[6],
+                    chunks: file[3],
+                    key: decrypted.key,
+                    bucket: file[1],
+                    region: file[2],
+                    creation: decrypted.creation,
+                    hash: decrypted.hash,
+                    path: filePath
+                };
+                tree[filePath] = item;
+                uuids[item.uuid] = item;
+            }
+            catch (e) {
+                this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
+            }
+        }));
+        this.previousTreeRawResponse = rawEx.length === 2 ? (_a = rawEx[0]) !== null && _a !== void 0 ? _a : "" : "";
         this.getDirectoryTreeCache = {
             timestamp: Date.now(),
             tree,
             uuids
         };
-        return { tree, uuids };
+        return {
+            result: {
+                tree,
+                uuids
+            },
+            ignored,
+            changed: true
+        };
     }
     /**
      * Find the corresponding UUID of the relative path.
@@ -90,6 +325,7 @@ class RemoteFileSystem {
             const basename = path_1.default.posix.basename(relativePath);
             if (parentPath === "/" || parentPath === "." || parentPath.length <= 0) {
                 const uuid = await this.sync.sdk.cloud().createDirectory({ name: basename, parent: this.sync.syncPair.remoteParentUUID });
+                await this.itemsMutex.acquire();
                 this.getDirectoryTreeCache.tree[relativePath] = {
                     type: "directory",
                     uuid,
@@ -97,6 +333,14 @@ class RemoteFileSystem {
                     size: 0,
                     path: relativePath
                 };
+                this.getDirectoryTreeCache.uuids[uuid] = {
+                    type: "directory",
+                    uuid,
+                    name: basename,
+                    size: 0,
+                    path: relativePath
+                };
+                this.itemsMutex.release();
                 return uuid;
             }
             const pathEx = relativePath.split("/");
@@ -116,6 +360,7 @@ class RemoteFileSystem {
                     const parentIsBase = partParentPath === "/" || partParentPath === "." || partParentPath === "";
                     const parentUUID = parentIsBase ? this.sync.syncPair.remoteParentUUID : parentItem.uuid;
                     const uuid = await this.sync.sdk.cloud().createDirectory({ name: partBasename, parent: parentUUID });
+                    await this.itemsMutex.acquire();
                     this.getDirectoryTreeCache.tree[relativePath] = {
                         type: "directory",
                         uuid,
@@ -123,6 +368,14 @@ class RemoteFileSystem {
                         size: 0,
                         path: relativePath
                     };
+                    this.getDirectoryTreeCache.uuids[uuid] = {
+                        type: "directory",
+                        uuid,
+                        name: partBasename,
+                        size: 0,
+                        path: relativePath
+                    };
+                    this.itemsMutex.release();
                 }
             }
             if (!this.getDirectoryTreeCache.tree[relativePath]) {
@@ -150,14 +403,15 @@ class RemoteFileSystem {
         await this.mutex.acquire();
         try {
             const uuid = await this.pathToItemUUID({ relativePath });
-            if (!uuid || !this.getDirectoryTreeCache.tree[relativePath]) {
+            const item = this.getDirectoryTreeCache.tree[relativePath];
+            if (!uuid || !item) {
                 return;
             }
             const acceptedTypes = !type ? ["directory", "file"] : type === "directory" ? ["directory"] : ["file"];
-            if (!acceptedTypes.includes(this.getDirectoryTreeCache.tree[relativePath].type)) {
+            if (!acceptedTypes.includes(item.type)) {
                 return;
             }
-            if (this.getDirectoryTreeCache.tree[relativePath].type === "directory") {
+            if (item.type === "directory") {
                 if (permanent) {
                     await this.sync.sdk.cloud().deleteDirectory({ uuid });
                 }
@@ -173,19 +427,26 @@ class RemoteFileSystem {
                     await this.sync.sdk.cloud().trashFile({ uuid });
                 }
             }
+            await this.itemsMutex.acquire();
             delete this.getDirectoryTreeCache.tree[relativePath];
+            delete this.getDirectoryTreeCache.uuids[uuid];
             for (const entry in this.getDirectoryTreeCache.tree) {
-                if (entry.startsWith(relativePath + "/")) {
+                if (entry.startsWith(relativePath + "/") || entry === relativePath) {
+                    const entryItem = this.getDirectoryTreeCache.tree[entry];
+                    if (entryItem) {
+                        delete this.getDirectoryTreeCache.uuids[entryItem.uuid];
+                    }
                     delete this.getDirectoryTreeCache.tree[entry];
                 }
             }
+            this.itemsMutex.release();
         }
         finally {
             this.mutex.release();
         }
     }
     /**
-     * Rename a file/directory inside the remote sync path. Recursively creates intermediate directories if needed.
+     * Rename/Move a file/directory inside the remote sync path. Recursively creates intermediate directories if needed.
      * @date 3/2/2024 - 9:35:12 PM
      *
      * @public
@@ -228,7 +489,10 @@ class RemoteFileSystem {
                     return;
                 }
                 if (item.type === "directory") {
-                    await this.sync.sdk.cloud().renameDirectory({ uuid, name: newBasename });
+                    await this.sync.sdk.cloud().renameDirectory({
+                        uuid,
+                        name: newBasename
+                    });
                 }
                 else {
                     await this.sync.sdk.cloud().renameFile({
@@ -237,16 +501,17 @@ class RemoteFileSystem {
                         name: newBasename
                     });
                 }
-                const oldItem = this.getDirectoryTreeCache.tree[fromRelativePath];
-                if (oldItem) {
-                    this.getDirectoryTreeCache.tree[toRelativePath] = Object.assign(Object.assign({}, oldItem), { name: newBasename });
-                }
-                delete this.getDirectoryTreeCache.tree[fromRelativePath];
             }
             else {
+                if (toRelativePath.startsWith(fromRelativePath)) {
+                    return;
+                }
                 if (oldBasename !== newBasename) {
                     if (item.type === "directory") {
-                        await this.sync.sdk.cloud().renameDirectory({ uuid, name: newBasename });
+                        await this.sync.sdk.cloud().renameDirectory({
+                            uuid,
+                            name: newBasename
+                        });
                     }
                     else {
                         await this.sync.sdk.cloud().renameFile({
@@ -258,14 +523,18 @@ class RemoteFileSystem {
                 }
                 if (newParentPath === "/" || newParentPath === "." || newParentPath === "") {
                     if (item.type === "directory") {
-                        await this.sync.sdk
-                            .cloud()
-                            .moveDirectory({ uuid, to: this.sync.syncPair.remoteParentUUID, metadata: itemMetadata });
+                        await this.sync.sdk.cloud().moveDirectory({
+                            uuid,
+                            to: this.sync.syncPair.remoteParentUUID,
+                            metadata: itemMetadata
+                        });
                     }
                     else {
-                        await this.sync.sdk
-                            .cloud()
-                            .moveFile({ uuid, to: this.sync.syncPair.remoteParentUUID, metadata: itemMetadata });
+                        await this.sync.sdk.cloud().moveFile({
+                            uuid,
+                            to: this.sync.syncPair.remoteParentUUID,
+                            metadata: itemMetadata
+                        });
                     }
                 }
                 else {
@@ -275,29 +544,39 @@ class RemoteFileSystem {
                         throw new Error(`Could not find path ${newParentPath}.`);
                     }
                     if (item.type === "directory") {
-                        await this.sync.sdk
-                            .cloud()
-                            .moveDirectory({ uuid, to: newParentItem.uuid, metadata: itemMetadata });
+                        await this.sync.sdk.cloud().moveDirectory({
+                            uuid,
+                            to: newParentItem.uuid,
+                            metadata: itemMetadata
+                        });
                     }
                     else {
-                        await this.sync.sdk.cloud().moveFile({ uuid, to: newParentItem.uuid, metadata: itemMetadata });
+                        await this.sync.sdk.cloud().moveFile({
+                            uuid,
+                            to: newParentItem.uuid,
+                            metadata: itemMetadata
+                        });
                     }
                 }
-                const oldItem = this.getDirectoryTreeCache.tree[fromRelativePath];
-                if (oldItem) {
-                    this.getDirectoryTreeCache.tree[toRelativePath] = Object.assign(Object.assign({}, oldItem), { name: newBasename });
-                }
+                await this.itemsMutex.acquire();
+                this.getDirectoryTreeCache.tree[toRelativePath] = Object.assign(Object.assign({}, item), { name: path_1.default.basename(toRelativePath), path: toRelativePath });
+                this.getDirectoryTreeCache.uuids[item.uuid] = Object.assign(Object.assign({}, item), { name: path_1.default.basename(toRelativePath), path: toRelativePath });
                 delete this.getDirectoryTreeCache.tree[fromRelativePath];
                 for (const oldPath in this.getDirectoryTreeCache.tree) {
-                    if (oldPath.startsWith(fromRelativePath + "/")) {
+                    if (oldPath.startsWith(fromRelativePath + "/") && oldPath !== fromRelativePath) {
                         const newPath = oldPath.split(fromRelativePath).join(toRelativePath);
                         const oldItem = this.getDirectoryTreeCache.tree[oldPath];
                         if (oldItem) {
-                            this.getDirectoryTreeCache.tree[newPath] = Object.assign(Object.assign({}, oldItem), { name: newBasename });
+                            this.getDirectoryTreeCache.tree[newPath] = Object.assign(Object.assign({}, oldItem), { name: path_1.default.basename(newPath), path: newPath });
+                            delete this.getDirectoryTreeCache.tree[oldPath];
+                            const oldItemUUID = this.getDirectoryTreeCache.uuids[oldItem.uuid];
+                            if (oldItemUUID) {
+                                this.getDirectoryTreeCache.uuids[oldItem.uuid] = Object.assign(Object.assign({}, oldItemUUID), { name: path_1.default.basename(newPath), path: newPath });
+                            }
                         }
-                        delete this.getDirectoryTreeCache.tree[oldPath];
                     }
                 }
+                this.itemsMutex.release();
             }
         }
         finally {
@@ -315,28 +594,123 @@ class RemoteFileSystem {
      * @returns {Promise<fs.Stats>}
      */
     async download({ relativePath }) {
+        var _a;
         const localPath = path_1.default.posix.join(this.sync.syncPair.localPath, relativePath);
-        const uuid = await this.pathToItemUUID({ relativePath });
-        const item = this.getDirectoryTreeCache.tree[relativePath];
-        if (!uuid || !item) {
-            throw new Error(`Could not download ${relativePath}: File not found.`);
+        const tmpLocalPath = path_1.default.join(this.sync.syncPair.localPath, constants_1.LOCAL_TRASH_NAME, (0, uuid_1.v4)());
+        const signalKey = `upload:${relativePath}`;
+        if (!this.sync.pauseSignals[signalKey]) {
+            this.sync.pauseSignals[signalKey] = new sdk_1.PauseSignal();
         }
-        if (item.type === "directory") {
-            throw new Error(`Could not download ${relativePath}: Not a file.`);
+        if (!this.sync.abortControllers[signalKey]) {
+            this.sync.abortControllers[signalKey] = new AbortController();
         }
-        const tmpPath = await this.sync.sdk.cloud().downloadFileToLocal({
-            uuid,
-            bucket: item.bucket,
-            region: item.region,
-            chunks: item.chunks,
-            version: item.version,
-            key: item.key
-        });
-        await fs_extra_1.default.move(tmpPath, localPath, {
-            overwrite: true
+        (0, ipc_1.postMessageToMain)({
+            type: "transfer",
+            syncPair: this.sync.syncPair,
+            data: {
+                of: "download",
+                type: "queued",
+                relativePath,
+                localPath
+            }
         });
-        await fs_extra_1.default.utimes(localPath, Date.now(), item.lastModified);
-        return await fs_extra_1.default.stat(localPath);
+        try {
+            const uuid = await this.pathToItemUUID({ relativePath });
+            const item = this.getDirectoryTreeCache.tree[relativePath];
+            if (!uuid || !item) {
+                throw new Error(`Could not download ${relativePath}: File not found.`);
+            }
+            if (item.type === "directory") {
+                throw new Error(`Could not download ${relativePath}: Not a file.`);
+            }
+            await this.sync.sdk.cloud().downloadFileToLocal({
+                uuid,
+                bucket: item.bucket,
+                region: item.region,
+                chunks: item.chunks,
+                version: item.version,
+                to: tmpLocalPath,
+                key: item.key,
+                size: item.size,
+                pauseSignal: this.sync.pauseSignals[signalKey],
+                abortSignal: (_a = this.sync.abortControllers[signalKey]) === null || _a === void 0 ? void 0 : _a.signal,
+                onError: err => {
+                    this.sync.worker.logger.log("error", err, "filesystems.remote.download");
+                    (0, ipc_1.postMessageToMain)({
+                        type: "transfer",
+                        syncPair: this.sync.syncPair,
+                        data: {
+                            of: "download",
+                            type: "error",
+                            relativePath,
+                            localPath,
+                            error: (0, utils_1.serializeError)(err)
+                        }
+                    });
+                },
+                onProgress: bytes => {
+                    (0, ipc_1.postMessageToMain)({
+                        type: "transfer",
+                        syncPair: this.sync.syncPair,
+                        data: {
+                            of: "download",
+                            type: "progress",
+                            relativePath,
+                            localPath,
+                            bytes
+                        }
+                    });
+                },
+                onStarted: () => {
+                    (0, ipc_1.postMessageToMain)({
+                        type: "transfer",
+                        syncPair: this.sync.syncPair,
+                        data: {
+                            of: "download",
+                            type: "started",
+                            relativePath,
+                            localPath
+                        }
+                    });
+                }
+            });
+            await fs_extra_1.default.utimes(tmpLocalPath, new Date(), new Date((0, utils_1.convertTimestampToMs)(item.lastModified)));
+            await fs_extra_1.default.move(tmpLocalPath, localPath, {
+                overwrite: true
+            });
+            (0, ipc_1.postMessageToMain)({
+                type: "transfer",
+                syncPair: this.sync.syncPair,
+                data: {
+                    of: "download",
+                    type: "finished",
+                    relativePath,
+                    localPath
+                }
+            });
+            return await fs_extra_1.default.stat(localPath);
+        }
+        catch (e) {
+            this.sync.worker.logger.log("error", e, "filesystems.remote.download");
+            if (e instanceof Error) {
+                (0, ipc_1.postMessageToMain)({
+                    type: "transfer",
+                    syncPair: this.sync.syncPair,
+                    data: {
+                        of: "download",
+                        type: "error",
+                        relativePath,
+                        localPath,
+                        error: (0, utils_1.serializeError)(e)
+                    }
+                });
+            }
+            throw e;
+        }
+        finally {
+            delete this.sync.pauseSignals[signalKey];
+            delete this.sync.abortControllers[signalKey];
+        }
     }
 }
 exports.RemoteFileSystem = RemoteFileSystem;