core-3nweb-client-lib 0.26.1 → 0.27.3

package/build/core/storage/synced/obj-files.js

@@ -16,30 +16,37 @@
  this program. If not, see <http://www.gnu.org/licenses/>.
 */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SyncedObj = exports.ObjFiles = void 0;
+exports.SyncedObj = exports.ObjFiles = exports.REMOTE_FILE_NAME_EXT = exports.UNSYNCED_FILE_NAME_EXT = void 0;
 const rxjs_1 = require("rxjs");
 const synced_1 = require("../../../lib-common/processes/synced");
-const sleep_1 = require("../../../lib-common/processes/sleep");
 const obj_folders_1 = require("../../../lib-client/objs-on-disk/obj-folders");
 const fs = require("../../../lib-common/async-fs-node");
 const obj_on_disk_1 = require("../../../lib-client/objs-on-disk/obj-on-disk");
 const path_1 = require("path");
+const downloader_1 = require("./downloader");
 const assert_1 = require("../../../lib-common/assert");
 const operators_1 = require("rxjs/operators");
 const utils_for_observables_1 = require("../../../lib-common/utils-for-observables");
 const obj_files_gc_1 = require("./obj-files-gc");
 const obj_status_1 = require("./obj-status");
 const timed_cache_1 = require("../../../lib-common/timed-cache");
-const UNSYNCED_FILE_NAME_EXT = 'unsynced';
-const SYNCED_FILE_NAME_EXT = 'v';
-const REMOTE_FILE_NAME_EXT = 'remote';
+const upload_header_file_1 = require("./upload-header-file");
+const utils_1 = require("../common/utils");
+exports.UNSYNCED_FILE_NAME_EXT = 'unsynced';
+exports.REMOTE_FILE_NAME_EXT = 'v';
+/**
+ * File system has nodes. Each node may have data in one or many objects of
+ * storage. SyncedObj allows to work with files of storage object, even when
+ * file system node no longer exists. ObjFiles is a holder and factory of
+ * SyncedObj's.
+ */
 class ObjFiles {
-    constructor(folders, downloader, logError) {
+    constructor(folders, remote, logError) {
         this.folders = folders;
-        this.downloader = downloader;
         this.logError = logError;
-        this.objs = timed_cache_1.makeTimedCache(60 * 1000);
+        this.objs = (0, timed_cache_1.makeTimedCache)(60 * 1000);
         this.sync = makeSynchronizer();
+        this.downloader = new downloader_1.Downloader(remote);
         this.gc = new obj_files_gc_1.GC(this.sync, obj => {
             if (this.objs.get(obj.objId) === obj) {
                 this.objs.delete(obj.objId);
@@ -47,29 +54,28 @@ class ObjFiles {
         }, objId => this.folders.removeFolderOf(objId));
         Object.freeze(this);
     }
-    static async makeFor(path, downloader, logError) {
-        const folders = await obj_folders_1.ObjFolders.makeWithGenerations(path, async (objId, objFolderPath) => {
-            if (objFiles.objs.has(objId)) {
+    static async makeFor(path, remote, logError) {
+        const canMove = (objId, objFolderPath) => objFiles.canMoveObjToDeeperCache(objId, objFolderPath);
+        const folders = await obj_folders_1.ObjFolders.makeWithGenerations(path, canMove, logError);
+        const objFiles = new ObjFiles(folders, remote, logError);
+        return objFiles;
+    }
+    async canMoveObjToDeeperCache(objId, objFolderPath) {
+        if (this.objs.has(objId)) {
+            return false;
+        }
+        const lst = await fs.readdir(objFolderPath);
+        for (const fName of lst) {
+            if (fName.endsWith(exports.UNSYNCED_FILE_NAME_EXT)) {
                 return false;
             }
-            const lst = await fs.readdir(objFolderPath);
-            for (const fName of lst) {
-                if (fName.endsWith(UNSYNCED_FILE_NAME_EXT)) {
-                    return false;
-                }
-                if (fName === obj_status_1.STATUS_FILE_NAME) {
-                    if (!objFiles.objs.has(objId)) {
-                        const status = await obj_status_1.readAndCheckStatus(objFolderPath, objId).catch(noop);
-                        if (status && status.syncTasks) {
-                            return false;
-                        }
-                    }
-                }
-            }
-            return true;
-        }, logError);
-        const objFiles = new ObjFiles(folders, downloader, logError);
-        return objFiles;
+        }
+        try {
+            return (await obj_status_1.ObjStatus.fileShowsObjNotInSyncedState(objFolderPath, objId));
+        }
+        catch (exc) {
+            return false;
+        }
     }
     async findObj(objId) {
         let obj = this.objs.get(objId);
@@ -86,12 +92,15 @@ class ObjFiles {
             return obj;
         });
     }
+    getObjInCache(objId) {
+        return this.objs.get(objId);
+    }
     makeObj(objId, download) {
         return this.sync(objId, async () => {
             const folder = await this.folders.getFolderAccessFor(objId, true);
             let obj;
             if (download) {
-                assert_1.assert(typeof download.version === 'number');
+                (0, assert_1.assert)(typeof download.version === 'number');
                 obj = await SyncedObj.forDownloadedObj(objId, folder, this.downloader, this.gc.scheduleCollection, download.version, download.parts, this.logError);
             }
             else {
@@ -112,6 +121,10 @@ class ObjFiles {
             return;
         });
     }
+    // XXX we'll need getting archived version.
+    //     Getting current may be changed to grub info about all obj versions.
+    //     As currently this.makeByDownloadingCurrentVersion() makes assumptions.
+    //     Should it be a pattern: version not in status -> get info first.
     async makeByDownloadingCurrentVersion(objId) {
         // initial download implicitly checks existence of obj on server
         const download = await this.downloader.getCurrentObjVersion(objId);
@@ -121,28 +134,29 @@ class ObjFiles {
     async saveFirstVersion(objId, encSub) {
         const newObj = await this.makeObj(objId);
         const fileWrite$ = (await newObj.saveNewVersion(1, encSub)).fileWrite$
-            .pipe(utils_for_observables_1.flatTap(undefined, err => this.removeFailedNewObj(newObj)));
+            .pipe((0, utils_for_observables_1.flatTap)(undefined, err => this.removeFailedNewObj(newObj)));
         return { fileWrite$, newObj };
     }
-    collectUnsyncedObjs() {
-        return rxjs_1.from([undefined])
+    findObjsToRemoveOnRemote() {
+        return (0, rxjs_1.from)([undefined])
             .pipe(
         // listing recent folders, exactly once
-        operators_1.mergeMap(() => this.folders.listRecent()), 
+        (0, operators_1.mergeMap)(() => this.folders.listRecent()), 
         // flatten array and space it in time, to process folders one by one
-        operators_1.mergeMap(objsAndPaths => objsAndPaths), operators_1.filter(({ objId }) => !this.objs.has(objId)), operators_1.mergeMap(async (objsAndPaths) => {
-            await sleep_1.sleep(20);
-            return objsAndPaths;
-        }, 1), 
-        // check, emiting objId, if not synced, and undefined, if synced
-        operators_1.mergeMap(({ path, objId }) => this.sync(objId, async () => {
-            if (this.objs.has(objId)) {
-                return;
+        (0, operators_1.mergeMap)(objsAndPaths => objsAndPaths), (0, operators_1.mergeMap)(async ({ objId, path }) => {
+            const obj = this.objs.get(objId);
+            if (obj) {
+                return (obj.statusObj().needsRemovalOnRemote() ?
+                    objId : undefined);
             }
-            const notSynced = await obj_status_1.ObjStatus.fileShowsObjNotInSyncedState(path, objId)
-                .catch(notFoundOrReThrow).catch(this.logError);
-            return (notSynced ? objId : undefined);
-        })), operators_1.filter(objId => (objId !== undefined)));
+            else {
+                const needsRm = await obj_status_1.ObjStatus.fileShowsObjNeedsRemovalOnRemote(path, objId);
+                return (needsRm ? objId : undefined);
+            }
+        }, 1), (0, operators_1.filter)(objId => (objId !== undefined)));
+    }
+    scheduleGC(obj) {
+        this.gc.scheduleCollection(obj);
     }
 }
 exports.ObjFiles = ObjFiles;
@@ -175,47 +189,26 @@ class SyncedObj {
         this.status = status;
         this.downloader = downloader;
         this.scheduleGC = scheduleGC;
-        /**
-         * These versions are what simple programs see. It is a local opinion about
-         * object's versions. Versions here can be either synced, or unsynced.
-         */
-        this.verObjs = timed_cache_1.makeTimedCache(60 * 1000);
-        /**
-         * These are conflicting versions, coming from a server. Of course, universal
-         * truth is spread by server, but in situations of parallel changes, local
-         * version allows things to work, while conflicting version from server
-         * should be adopted by conflict resolution process. In other words, these
-         * versions are not for common use.
-         */
-        this.remoteConflictVerObjs = timed_cache_1.makeTimedCache(60 * 1000);
-        this.objSegsGetterFromDisk = async (ver, ofs, len) => {
-            let obj = this.verObjs.get(ver);
-            if (!obj) {
-                const fPath = this.path(ver, this.status.isVersionSynced(ver));
-                try {
-                    obj = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, ver, fPath, this.downloader, this.objSegsGetterFromDisk);
-                    this.verObjs.set(ver, obj);
-                }
-                catch (exc) {
-                    // when file doesn't exist on a disk, we just pass a chunk
-                    if (!exc.notFound) {
-                        throw exc;
-                    }
-                    return [{ type: 'new', thisVerOfs: ofs, len }];
-                }
+        this.remoteVers = (0, timed_cache_1.makeTimedCache)(60 * 1000);
+        this.localVers = (0, timed_cache_1.makeTimedCache)(60 * 1000);
+        this.localAndSyncedObjSegsGetterFromDisk = async (v, ofs, len) => {
+            const latestSynced = this.status.latestSyncedVersion();
+            if (latestSynced && (latestSynced >= v)) {
+                return this.remoteObjSegsGetterFromDisk(v, ofs, len);
             }
-            return obj.readSegsOnlyFromDisk(ofs, len);
-        };
-        this.conflictObjSegsGetterFromDisk = async (version, ofs, len) => {
-            if (this.status.syncedVersionGreaterOrEqual(version)) {
-                return this.objSegsGetterFromDisk(version, ofs, len);
+            let objVer = this.localVers.get(v);
+            if (!objVer) {
+                objVer = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, v, this.localVerPath(v), this.downloader, this.localAndSyncedObjSegsGetterFromDisk);
+                this.localVers.set(v, objVer);
             }
-            let obj = this.remoteConflictVerObjs.get(version);
-            if (!obj) {
-                const fPath = this.path(version, true);
+            return objVer.readSegsOnlyFromDisk(ofs, len);
+        };
+        this.remoteObjSegsGetterFromDisk = async (v, ofs, len) => {
+            let objVer = this.remoteVers.get(v);
+            if (!objVer) {
                 try {
-                    obj = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.conflictObjSegsGetterFromDisk);
-                    this.remoteConflictVerObjs.set(version, obj);
+                    objVer = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, v, this.remoteVerPath(v), this.downloader, this.remoteObjSegsGetterFromDisk);
+                    this.remoteVers.set(v, objVer);
                 }
                 catch (exc) {
                     // when file doesn't exist on a disk, we just pass a chunk
@@ -225,7 +218,7 @@ class SyncedObj {
                     return [{ type: 'new', thisVerOfs: ofs, len }];
                 }
             }
-            return obj.readSegsOnlyFromDisk(ofs, len);
+            return objVer.readSegsOnlyFromDisk(ofs, len);
         };
         Object.freeze(this);
     }
@@ -234,126 +227,173 @@ class SyncedObj {
         return new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
     }
     static async forDownloadedObj(objId, objFolder, downloader, scheduleGC, version, parts, logError) {
-        // XXX let's note that given version is used to be current on server
+        // XXX let's note that given version is also passed as current on server
         const status = await obj_status_1.ObjStatus.makeForDownloadedVersion(objFolder, objId, version, version, logError);
         const obj = new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
-        const fPath = obj.path(version, true);
-        const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.objSegsGetterFromDisk, parts);
-        obj.verObjs.set(version, objVer);
+        const fPath = obj.remoteVerPath(version);
+        const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk, parts);
+        obj.remoteVers.set(version, objVer);
         return obj;
     }
     static async forNewObj(objId, objFolder, downloader, scheduleGC, logError) {
         const status = await obj_status_1.ObjStatus.makeNew(objFolder, objId, logError);
         return new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
     }
-    sync() {
-        return this.status;
-    }
     scheduleSelfGC() {
         this.scheduleGC(this);
     }
-    path(version, synced, remote) {
-        const fileExt = (synced ?
-            SYNCED_FILE_NAME_EXT :
-            (remote ? REMOTE_FILE_NAME_EXT : UNSYNCED_FILE_NAME_EXT));
-        const fName = `${version}.${fileExt}`;
-        return path_1.join(this.objFolder, fName);
+    localVerPath(version) {
+        return (0, path_1.join)(this.objFolder, `${version}.${exports.UNSYNCED_FILE_NAME_EXT}`);
     }
-    async getObjSrc(version) {
-        let objVer = this.verObjs.get(version);
+    remoteVerPath(version) {
+        return (0, path_1.join)(this.objFolder, `${version}.${exports.REMOTE_FILE_NAME_EXT}`);
+    }
+    async getObjSrcFromLocalAndSyncedBranch(version) {
+        const latestSynced = this.status.latestSyncedVersion();
+        const objVer = await ((latestSynced && (latestSynced >= version)) ?
+            this.instanceOfRemoteObjVer(version) :
+            this.instanceOfLocalObjVer(version));
+        return objVer.getSrc();
+    }
+    async getObjSrcFromRemoteAndSyncedBranch(version) {
+        const objVer = await this.instanceOfRemoteObjVer(version);
+        return objVer.getSrc();
+    }
+    async instanceOfLocalObjVer(version) {
+        let objVer = this.localVers.get(version);
         if (objVer) {
-            return objVer.getSrc();
-        }
-        const isSynced = this.status.isVersionSynced(version);
-        const fPath = this.path(version, isSynced);
-        if (isSynced) {
-            objVer = ((await isOnDisk(fPath)) ?
-                await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.objSegsGetterFromDisk) :
-                await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(this.objId, version, fPath, this.downloader, this.objSegsGetterFromDisk));
+            return objVer;
         }
-        else {
-            objVer = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.objSegsGetterFromDisk);
-        }
-        const src = objVer.getSrc();
-        this.verObjs.set(version, objVer);
-        return src;
+        const fPath = this.localVerPath(version);
+        objVer = await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.localAndSyncedObjSegsGetterFromDisk);
+        this.localVers.set(version, objVer);
+        return objVer;
     }
-    async getRemoteConflictObjVersion(version) {
-        if (this.status.syncedVersionGreaterOrEqual(version)) {
-            return this.getObjSrc(version);
-        }
-        let obj = this.remoteConflictVerObjs.get(version);
-        if (obj) {
-            return obj.getSrc();
+    async instanceOfRemoteObjVer(version) {
+        let objVer = this.remoteVers.get(version);
+        if (objVer) {
+            return objVer;
         }
-        const fPath = this.path(version, true);
-        obj = ((await isOnDisk(fPath)) ?
-            await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.conflictObjSegsGetterFromDisk) :
-            await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(this.objId, version, fPath, this.downloader, this.conflictObjSegsGetterFromDisk));
-        const src = obj.getSrc();
-        this.verObjs.set(version, obj);
-        return src;
+        const fPath = this.remoteVerPath(version);
+        objVer = ((await isOnDisk(fPath)) ?
+            await obj_on_disk_1.ObjOnDisk.forExistingFile(this.objId, version, fPath, this.downloader, this.remoteObjSegsGetterFromDisk) :
+            await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(this.objId, version, fPath, this.downloader, this.remoteObjSegsGetterFromDisk));
+        this.remoteVers.set(version, objVer);
+        return objVer;
     }
     async saveNewVersion(version, encSub) {
-        if (this.verObjs.has(version)) {
+        if (this.localVers.has(version)) {
             throw new Error(`Version ${version} already exists in object ${this.objId}`);
         }
-        const fPath = this.path(version, false);
-        const { obj, write$ } = await obj_on_disk_1.ObjOnDisk.createFileForWriteOfNewVersion(this.objId, version, fPath, encSub, this.downloader, this.objSegsGetterFromDisk);
-        this.verObjs.set(version, obj);
-        const fileWrite$ = write$.pipe(operators_1.tap({
+        const fPath = this.localVerPath(version);
+        const { obj, write$ } = await obj_on_disk_1.ObjOnDisk.createFileForWriteOfNewVersion(this.objId, version, fPath, encSub, this.downloader, this.localAndSyncedObjSegsGetterFromDisk);
+        this.localVers.set(version, obj);
+        const fileWrite$ = write$.pipe((0, operators_1.tap)({
             error: err => {
-                if (this.verObjs.get(version) === obj) {
-                    this.verObjs.delete(version);
+                if (this.localVers.get(version) === obj) {
+                    this.localVers.delete(version);
                 }
             }
-        }), utils_for_observables_1.flatTap(undefined, undefined, () => this.setUnsyncedCurrentVersion(version, obj.getBaseVersion())));
+        }), (0, utils_for_observables_1.flatTap)(undefined, undefined, () => this.setUnsyncedCurrentVersion(version, obj.getBaseVersion())));
         return { fileWrite$, baseVer: obj.getBaseVersion() };
     }
-    isArchived() {
-        return this.status.isArchived();
-    }
-    getCurrentVersionOrThrow() {
-        return this.status.getCurrentVersionOrThrow();
-    }
-    isRemoteVersionGreaterOrEqualTo(newRemoteVersion) {
-        return this.status.isRemoteVersionGreaterOrEqualTo(newRemoteVersion);
+    async combineLocalBaseIfPresent(version) {
+        const bases = this.status.baseOfLocalVersion(version);
+        if (!bases) {
+            return;
+        }
+        const { localBases, syncedBase } = bases;
+        if (localBases) {
+            const objVer = await this.instanceOfLocalObjVer(version);
+            for (const localBase of localBases) {
+                await objVer.absorbImmediateBaseVersion(localBase, this.localVerPath(localBase));
+                this.status.absorbLocalVersionBase(version, localBase);
+            }
+        }
+        return syncedBase;
     }
-    isDeletedOnRemote() {
-        return this.status.isDeletedOnRemote();
+    async saveUploadHeaderFile(uploadHeader) {
+        await (0, upload_header_file_1.saveUploadHeaderFile)(this.objFolder, uploadHeader);
     }
     async setUnsyncedCurrentVersion(version, baseVersion) {
-        await this.status.setUnsyncedCurrentVersion(version, baseVersion);
+        await this.status.setLocalCurrentVersion(version, baseVersion);
         if (version > 1) {
             this.scheduleSelfGC();
         }
     }
-    async markVersionSynced(version) {
-        const verObj = this.verObjs.get(version);
+    /**
+     * This renames/moves version file from local to remote.
+     * Removes upload info from status, updating enumerations of local and
+     * synced versions.
+     */
+    async recordUploadCompletion(localVersion, uploadVersion, headerChange) {
+        const verObj = this.localVers.get(localVersion);
+        const remotePath = this.remoteVerPath(uploadVersion);
         if (verObj) {
-            await verObj.moveFile(this.path(version, true));
+            const syncedVerObj = await verObj.moveFileAndProxyThis(remotePath, (headerChange ? {
+                newHeader: headerChange.newHeader,
+                originalHeader: headerChange.originalHeader,
+                version: uploadVersion
+            } : undefined));
+            this.remoteVers.set(uploadVersion, syncedVerObj);
         }
         else {
-            await fs.rename(this.path(version, false), this.path(version, true));
+            await fs.rename(this.localVerPath(localVersion), remotePath);
         }
-        await this.status.markVersionSynced(version);
+        await this.status.recordUploadCompletion(localVersion, uploadVersion);
         this.scheduleSelfGC();
     }
-    async setRemoteVersion(version) {
-        await this.status.setRemoteVersion(version);
+    dropCachedLocalObjVersionsLessOrEqual(version) {
+        for (const version of this.localVers.keys()) {
+            if (version <= version) {
+                this.localVers.delete(version);
+            }
+        }
     }
-    async setDeletedOnRemote() {
-        await this.status.setDeletedOnRemote();
+    async removeLocalVersionFilesLessThan(version) {
+        const lst = await fs.readdir(this.objFolder).catch(utils_1.noop);
+        if (!lst) {
+            return;
+        }
+        const rmProcs = [];
+        for (const f of lst) {
+            if (!f.endsWith(exports.UNSYNCED_FILE_NAME_EXT)) {
+                continue;
+            }
+            const verStr = f.slice(0, f.length - 1 - exports.UNSYNCED_FILE_NAME_EXT.length);
+            const ver = parseInt(verStr);
+            if (isNaN(ver) || (ver > version)) {
+                continue;
+            }
+            rmProcs.push(fs.unlink((0, path_1.join)(this.objFolder, f)).catch(utils_1.noop));
+        }
+        if (rmProcs.length > 0) {
+            await Promise.all(rmProcs);
+        }
     }
     async removeCurrentVersion() {
-        await this.status.removeCurrentVersion(this.verObjs);
+        await this.status.removeCurrentVersion();
+        // note that gc is tasked with removing current obj version on server
         this.scheduleSelfGC();
     }
-    getNonGarbageVersions() {
-        return this.status.getNonGarbageVersions();
+    async diffForUploadOf(version) {
+        const objVer = await this.instanceOfLocalObjVer(version);
+        if (objVer.getBaseVersion()) {
+            return objVer.diffFromBase();
+        }
+        else {
+            throw new Error(`Version ${version} is not a diff version`);
+        }
+    }
+    syncStatus() {
+        return this.status;
     }
-    isStatusFileSaved() {
-        return this.status.isFileSaved();
+    statusObj() {
+        return this.status;
+    }
+    async recordRemovalUploadAndGC() {
+        await this.status.recordRemoteRemovalCompletion();
+        this.scheduleSelfGC();
     }
 }
 exports.SyncedObj = SyncedObj;
@@ -368,5 +408,4 @@ async function isOnDisk(path) {
         throw exc;
     }));
 }
-function noop() { }
 Object.freeze(exports);