core-3nweb-client-lib 0.25.3 → 0.26.0

package/build/core/storage/synced/obj-files-gc.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2020 3NSoft Inc.
+ Copyright (C) 2020, 2022 3NSoft Inc.
 
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -13,10 +13,13 @@
  See the GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GC = void 0;
-const processes_1 = require("../../../lib-common/processes");
+const synced_1 = require("../../../lib-common/processes/synced");
+const fs = require("../../../lib-common/async-fs-node");
+const path_1 = require("path");
 class GC {
     constructor(sync, rmObjFromCache, rmObjFolder) {
         this.sync = sync;
@@ -26,7 +29,7 @@ class GC {
         /**
          * All gc steps are done in this process.
          */
-        this.gcProc = new processes_1.SingleProc();
+        this.gcProc = new synced_1.SingleProc();
         /**
          * wip are objects that are currently processed. When wip set is empty,
          * it gets swapped with non-empty scheduled set.
@@ -55,7 +58,7 @@ class GC {
             if (!obj) {
                 return;
             }
-            // XXX
+            await this.sync(obj.objId, () => this.collectIn(obj).catch(noop));
             return this.objCollecting();
         };
         Object.seal(this);
@@ -63,6 +66,41 @@ class GC {
     async stop() {
         this.isStopped = true;
     }
+    async collectIn(obj) {
+        // calculate versions that should not be removed
+        const { gcMaxVer, nonGarbage } = obj.getNonGarbageVersions();
+        // if object is set archived, and there is nothing in it worth keeping,
+        // whole folder can be removed
+        if ((nonGarbage.size === 0)
+            && obj.isArchived() && obj.sync().isSyncDone()) {
+            if (!obj.isStatusFileSaved()) {
+                return;
+            }
+            this.rmObjFromCache(obj);
+            if (obj.objId) {
+                await this.rmObjFolder(obj.objId);
+            }
+            return;
+        }
+        // for all other cases, we remove version files that are not worth
+        // keeping.
+        const lst = await fs.readdir(obj.objFolder).catch(noop);
+        if (!lst) {
+            return;
+        }
+        const rmProcs = [];
+        for (const f of lst) {
+            const ver = parseInt(f);
+            if (isNaN(ver) || nonGarbage.has(ver)
+                || (gcMaxVer && (ver >= gcMaxVer))) {
+                continue;
+            }
+            rmProcs.push(fs.unlink(path_1.join(obj.objFolder, f)).catch(noop));
+        }
+        if (rmProcs.length > 0) {
+            await Promise.all(rmProcs);
+        }
+    }
 }
 exports.GC = GC;
 Object.freeze(GC.prototype);
@@ -76,4 +114,5 @@ function getAndRemoveOneFrom(set) {
     set.delete(value);
     return value;
 }
+function noop() { }
 Object.freeze(exports);

package/build/core/storage/synced/obj-files.d.ts

@@ -5,22 +5,8 @@ import { ObjSource, Subscribe } from 'xsp-files';
 import { Downloader } from './downloader';
 import { FileWrite } from '../../../lib-client/objs-on-disk/file-writing-proc';
 import { GC } from './obj-files-gc';
+import { UpSyncStatus } from './obj-status';
 import { LogError } from '../../../lib-client/logging/log-to-file';
-/**
- * This is an extention for an unsynced version file. File name starts with a
- * version number.
- */
-export declare const UNSYNCED_FILE_NAME_EXT = "unsynced";
-/**
- * This is an upload info file. When an upload is complete,
- * this info file is removed.
- */
-export declare const UPLOAD_INFO_FNAME = "upload";
-/**
- * Presence of this empty file in a object folder indicates that object's
- * removal hasn't been synchronized, yet.
- */
-export declare const UNSYNCED_REMOVAL = "unsynced-removal";
 export declare class ObjFiles {
     private readonly folders;
     private readonly downloader;
@@ -61,9 +47,11 @@ export declare class SyncedObj {
      */
     private readonly remoteConflictVerObjs;
     private constructor();
-    static forExistingObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection']): Promise<SyncedObj>;
-    static forDownloadedObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection'], version: number, parts: InitDownloadParts): Promise<SyncedObj>;
-    static forNewObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection']): Promise<SyncedObj>;
+    static forExistingObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection'], logError: LogError): Promise<SyncedObj>;
+    static forDownloadedObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection'], version: number, parts: InitDownloadParts, logError: LogError): Promise<SyncedObj>;
+    static forNewObj(objId: ObjId, objFolder: string, downloader: Downloader, scheduleGC: GC['scheduleCollection'], logError: LogError): Promise<SyncedObj>;
+    sync(): UpSyncStatus;
+    scheduleSelfGC(): void;
     private path;
     getObjSrc(version: number): Promise<ObjSource>;
     private objSegsGetterFromDisk;
@@ -77,9 +65,14 @@ export declare class SyncedObj {
     getCurrentVersionOrThrow(): number;
     isRemoteVersionGreaterOrEqualTo(newRemoteVersion: number): boolean;
     isDeletedOnRemote(): boolean;
-    setUnsyncedCurrentVersion(version: number): Promise<void>;
-    setSyncedCurrentVersion(version: number): Promise<void>;
+    private setUnsyncedCurrentVersion;
+    markVersionSynced(version: number): Promise<void>;
     setRemoteVersion(version: number): Promise<void>;
     setDeletedOnRemote(): Promise<void>;
     removeCurrentVersion(): Promise<void>;
+    getNonGarbageVersions(): {
+        gcMaxVer?: number;
+        nonGarbage: Set<number>;
+    };
+    isStatusFileSaved(): boolean;
 }

package/build/core/storage/synced/obj-files.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2016 - 2020 3NSoft Inc.
+ Copyright (C) 2016 - 2020, 2022 3NSoft Inc.
 
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -16,9 +16,10 @@
  this program. If not, see <http://www.gnu.org/licenses/>.
 */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SyncedObj = exports.ObjFiles = exports.UNSYNCED_REMOVAL = exports.UPLOAD_INFO_FNAME = exports.UNSYNCED_FILE_NAME_EXT = void 0;
+exports.SyncedObj = exports.ObjFiles = void 0;
 const rxjs_1 = require("rxjs");
-const processes_1 = require("../../../lib-common/processes");
+const synced_1 = require("../../../lib-common/processes/synced");
+const sleep_1 = require("../../../lib-common/processes/sleep");
 const obj_folders_1 = require("../../../lib-client/objs-on-disk/obj-folders");
 const fs = require("../../../lib-common/async-fs-node");
 const obj_on_disk_1 = require("../../../lib-client/objs-on-disk/obj-on-disk");
@@ -29,21 +30,9 @@ const utils_for_observables_1 = require("../../../lib-common/utils-for-observabl
 const obj_files_gc_1 = require("./obj-files-gc");
 const obj_status_1 = require("./obj-status");
 const timed_cache_1 = require("../../../lib-common/timed-cache");
-/**
- * This is an extention for an unsynced version file. File name starts with a
- * version number.
- */
-exports.UNSYNCED_FILE_NAME_EXT = 'unsynced';
-/**
- * This is an upload info file. When an upload is complete,
- * this info file is removed.
- */
-exports.UPLOAD_INFO_FNAME = 'upload';
-/**
- * Presence of this empty file in a object folder indicates that object's
- * removal hasn't been synchronized, yet.
- */
-exports.UNSYNCED_REMOVAL = 'unsynced-removal';
+const UNSYNCED_FILE_NAME_EXT = 'unsynced';
+const SYNCED_FILE_NAME_EXT = 'v';
+const REMOTE_FILE_NAME_EXT = 'remote';
 class ObjFiles {
     constructor(folders, downloader, logError) {
         this.folders = folders;
@@ -65,11 +54,17 @@ class ObjFiles {
             }
             const lst = await fs.readdir(objFolderPath);
             for (const fName of lst) {
-                if (fName.endsWith(exports.UNSYNCED_FILE_NAME_EXT)
-                    || (fName === exports.UPLOAD_INFO_FNAME)
-                    || (fName === exports.UNSYNCED_REMOVAL)) {
+                if (fName.endsWith(UNSYNCED_FILE_NAME_EXT)) {
                     return false;
                 }
+                if (fName === obj_status_1.STATUS_FILE_NAME) {
+                    if (!objFiles.objs.has(objId)) {
+                        const status = await obj_status_1.readAndCheckStatus(objFolderPath, objId).catch(noop);
+                        if (status && status.syncTasks) {
+                            return false;
+                        }
+                    }
+                }
             }
             return true;
         }, logError);
@@ -86,7 +81,7 @@ class ObjFiles {
             if (!folder) {
                 return;
             }
-            const obj = await SyncedObj.forExistingObj(objId, folder, this.downloader, this.gc.scheduleCollection);
+            const obj = await SyncedObj.forExistingObj(objId, folder, this.downloader, this.gc.scheduleCollection, this.logError);
             this.objs.set(objId, obj);
             return obj;
         });
@@ -97,10 +92,10 @@ class ObjFiles {
             let obj;
             if (download) {
                 assert_1.assert(typeof download.version === 'number');
-                obj = await SyncedObj.forDownloadedObj(objId, folder, this.downloader, this.gc.scheduleCollection, download.version, download.parts);
+                obj = await SyncedObj.forDownloadedObj(objId, folder, this.downloader, this.gc.scheduleCollection, download.version, download.parts, this.logError);
             }
             else {
-                obj = await SyncedObj.forNewObj(objId, folder, this.downloader, this.gc.scheduleCollection);
+                obj = await SyncedObj.forNewObj(objId, folder, this.downloader, this.gc.scheduleCollection, this.logError);
             }
             this.objs.set(objId, obj);
             return obj;
@@ -136,7 +131,7 @@ class ObjFiles {
         operators_1.mergeMap(() => this.folders.listRecent()), 
         // flatten array and space it in time, to process folders one by one
         operators_1.mergeMap(objsAndPaths => objsAndPaths), operators_1.filter(({ objId }) => !this.objs.has(objId)), operators_1.mergeMap(async (objsAndPaths) => {
-            await processes_1.sleep(20);
+            await sleep_1.sleep(20);
             return objsAndPaths;
         }, 1), 
         // check, emiting objId, if not synced, and undefined, if synced
@@ -155,10 +150,10 @@ Object.freeze(ObjFiles.prototype);
 Object.freeze(ObjFiles);
 function makeSynchronizer(proc) {
     if (!proc) {
-        proc = new processes_1.NamedProcs();
+        proc = new synced_1.NamedProcs();
     }
     async function sync(objId, action) {
-        const id = (!objId ? '==root==' : objId);
+        const id = (objId ? objId : '==root==');
         return proc.startOrChain(id, action);
     }
     return sync;
@@ -234,25 +229,34 @@ class SyncedObj {
         };
         Object.freeze(this);
     }
-    static async forExistingObj(objId, objFolder, downloader, scheduleGC) {
-        const status = await obj_status_1.ObjStatus.readFrom(objFolder, objId);
+    static async forExistingObj(objId, objFolder, downloader, scheduleGC, logError) {
+        const status = await obj_status_1.ObjStatus.readFrom(objFolder, objId, logError);
         return new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
     }
-    static async forDownloadedObj(objId, objFolder, downloader, scheduleGC, version, parts) {
-        const status = await obj_status_1.ObjStatus.makeForDownloadedVersion(objFolder, objId, version);
+    static async forDownloadedObj(objId, objFolder, downloader, scheduleGC, version, parts, logError) {
+        // XXX let's note that given version is used to be current on server
+        const status = await obj_status_1.ObjStatus.makeForDownloadedVersion(objFolder, objId, version, version, logError);
         const obj = new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
-        const fPath = obj.path(version, false);
+        const fPath = obj.path(version, true);
         const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.objSegsGetterFromDisk, parts);
         obj.verObjs.set(version, objVer);
         return obj;
     }
-    static async forNewObj(objId, objFolder, downloader, scheduleGC) {
-        const status = await obj_status_1.ObjStatus.makeNew(objFolder, objId);
+    static async forNewObj(objId, objFolder, downloader, scheduleGC, logError) {
+        const status = await obj_status_1.ObjStatus.makeNew(objFolder, objId, logError);
         return new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
     }
-    path(version, synced) {
-        const fName = (synced ?
-            `${version}.v` : `${version}.${exports.UNSYNCED_FILE_NAME_EXT}`);
+    sync() {
+        return this.status;
+    }
+    scheduleSelfGC() {
+        this.scheduleGC(this);
+    }
+    path(version, synced, remote) {
+        const fileExt = (synced ?
+            SYNCED_FILE_NAME_EXT :
+            (remote ? REMOTE_FILE_NAME_EXT : UNSYNCED_FILE_NAME_EXT));
+        const fName = `${version}.${fileExt}`;
         return path_1.join(this.objFolder, fName);
     }
     async getObjSrc(version) {
@@ -297,11 +301,13 @@ class SyncedObj {
         const fPath = this.path(version, false);
         const { obj, write$ } = await obj_on_disk_1.ObjOnDisk.createFileForWriteOfNewVersion(this.objId, version, fPath, encSub, this.downloader, this.objSegsGetterFromDisk);
         this.verObjs.set(version, obj);
-        const fileWrite$ = write$.pipe(operators_1.tap(undefined, err => {
-            if (this.verObjs.get(version) === obj) {
-                this.verObjs.delete(version);
+        const fileWrite$ = write$.pipe(operators_1.tap({
+            error: err => {
+                if (this.verObjs.get(version) === obj) {
+                    this.verObjs.delete(version);
+                }
             }
-        }), utils_for_observables_1.flatTap(undefined, undefined, () => this.setUnsyncedCurrentVersion(version)));
+        }), utils_for_observables_1.flatTap(undefined, undefined, () => this.setUnsyncedCurrentVersion(version, obj.getBaseVersion())));
         return { fileWrite$, baseVer: obj.getBaseVersion() };
     }
     isArchived() {
@@ -316,13 +322,22 @@ class SyncedObj {
     isDeletedOnRemote() {
         return this.status.isDeletedOnRemote();
     }
-    async setUnsyncedCurrentVersion(version) {
-        await this.status.setUnsyncedCurrentVersion(version);
-        this.scheduleGC(this);
+    async setUnsyncedCurrentVersion(version, baseVersion) {
+        await this.status.setUnsyncedCurrentVersion(version, baseVersion);
+        if (version > 1) {
+            this.scheduleSelfGC();
+        }
     }
-    async setSyncedCurrentVersion(version) {
-        await this.status.setSyncedCurrentVersion(version);
-        this.scheduleGC(this);
+    async markVersionSynced(version) {
+        const verObj = this.verObjs.get(version);
+        if (verObj) {
+            await verObj.moveFile(this.path(version, true));
+        }
+        else {
+            await fs.rename(this.path(version, false), this.path(version, true));
+        }
+        await this.status.markVersionSynced(version);
+        this.scheduleSelfGC();
     }
     async setRemoteVersion(version) {
         await this.status.setRemoteVersion(version);
@@ -332,7 +347,13 @@ class SyncedObj {
     }
     async removeCurrentVersion() {
         await this.status.removeCurrentVersion(this.verObjs);
-        this.scheduleGC(this);
+        this.scheduleSelfGC();
+    }
+    getNonGarbageVersions() {
+        return this.status.getNonGarbageVersions();
+    }
+    isStatusFileSaved() {
+        return this.status.isFileSaved();
     }
 }
 exports.SyncedObj = SyncedObj;
@@ -347,4 +368,5 @@ async function isOnDisk(path) {
         throw exc;
     }));
 }
+function noop() { }
 Object.freeze(exports);

package/build/core/storage/synced/obj-status.d.ts

@@ -1,4 +1,7 @@
 import { ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
+import { VersionsInfo } from '../common/obj-info-file';
+import { LogError } from '../../../lib-client/logging/log-to-file';
+declare type Stats = web3n.files.Stats;
 /**
  * Storage object can be in following states:
  * 1. Synced state indicates that current local version and one on the server
@@ -14,45 +17,101 @@ export declare type SyncState = 'synced' | 'behind' | 'unsynced' | 'conflicting'
 export interface ObjStatusInfo {
     objId: ObjId;
     isArchived?: boolean;
-    deletedOnRemote?: boolean;
-    syncState: SyncState;
-    versions: {
-        latestSynced?: number;
+    versions: VersionsInfo;
+    sync: {
+        state: SyncState;
+        latest?: number;
         conflictingRemote?: number[];
-        current?: number;
-        archived?: number[];
-        unsynchedArchived?: number[];
         remote?: number;
+        deletedOnRemote?: true;
     };
-    gcWorkInfo?: object;
+    syncTasks?: {
+        queued: UpSyncTaskInfo[];
+        current?: UpSyncTaskInfo;
+    };
+}
+export interface UploadInfo {
+    type: 'upload';
+    transactionId?: string;
+    createObj?: true;
+    needUpload?: {
+        header?: number;
+        segs: BytesSection[];
+        allByteOnDisk?: true;
+    };
+    version: number;
+    baseVersion?: number;
+}
+export interface BytesSection {
+    ofs: number;
+    len: number;
 }
-export declare class ObjStatus {
+export interface RemovalInfo {
+    type: 'removal';
+    archivedVersions?: number[] | number;
+    currentVersion?: true;
+}
+export interface ArchivalInfo {
+    type: 'archiving';
+    archivalOfCurrent?: true;
+}
+export declare type UpSyncTaskInfo = UploadInfo | RemovalInfo | ArchivalInfo;
+export interface UpSyncStatus {
+    queueTask(t: UpSyncTaskInfo): void;
+    getTaskForProcessing(): Promise<UpSyncTaskInfo | undefined>;
+    glanceOnNextTask(): UpSyncTaskInfo | undefined;
+    recordInterimStateOfCurrentTask(t: UploadInfo): Promise<void>;
+    recordTaskCompletion(t: UpSyncTaskInfo): Promise<void>;
+    isSyncDone(): boolean;
+    stat(): NonNullable<Stats['sync']>;
+}
+export declare const STATUS_FILE_NAME = "status";
+export declare class ObjStatus implements UpSyncStatus {
     private readonly objFolder;
     private readonly status;
+    private readonly logError;
     private readonly saveProc;
     private constructor();
-    static readFrom(objFolder: string, objId: ObjId): Promise<ObjStatus>;
-    static makeNew(objFolder: string, objId: ObjId): Promise<ObjStatus>;
-    static makeForDownloadedVersion(objFolder: string, objId: ObjId, version: number): Promise<ObjStatus>;
+    static readFrom(objFolder: string, objId: ObjId, logError: LogError): Promise<ObjStatus>;
+    static makeNew(objFolder: string, objId: ObjId, logError: LogError): Promise<ObjStatus>;
+    static makeForDownloadedVersion(objFolder: string, objId: ObjId, version: number, currentRemote: number, logError: LogError): Promise<ObjStatus>;
     static fileShowsObjNotInSyncedState(objFolder: string, objId: ObjId): Promise<boolean>;
-    private saveFile;
     isArchived(): boolean;
     getCurrentVersionOrThrow(): number;
+    getNonGarbageVersions(): {
+        gcMaxVer?: number;
+        nonGarbage: Set<number>;
+    };
     isRemoteVersionGreaterOrEqualTo(newRemoteVersion: number): boolean;
     isDeletedOnRemote(): boolean;
     syncedVersionGreaterOrEqual(version: number): boolean;
     removeCurrentVersion(verObjs: ContainerWithDelete<number>): Promise<void>;
+    private triggerSaveProc;
     setDeletedOnRemote(): Promise<void>;
     setRemoteVersion(version: number): Promise<void>;
-    setSyncedCurrentVersion(version: number): Promise<void>;
-    setUnsyncedCurrentVersion(version: number): Promise<void>;
+    markVersionSynced(version: number): Promise<void>;
+    setUnsyncedCurrentVersion(version: number, baseVer: number | undefined): Promise<void>;
     /**
      * This method ignores remote conflicting versions.
      * @param version
      */
     isVersionSynced(version: number): boolean;
+    isVersionArchived(version: number): boolean;
+    queueTask(t: UpSyncTaskInfo): void;
+    private addUploadToQueue;
+    private addRemoveCurrentToQueue;
+    private addRemoveArchivedToQueue;
+    private addArchivalToQueue;
+    getTaskForProcessing(): Promise<UpSyncTaskInfo | undefined>;
+    glanceOnNextTask(): UpSyncTaskInfo | undefined;
+    recordInterimStateOfCurrentTask(t: UploadInfo): Promise<void>;
+    recordTaskCompletion(t: UpSyncTaskInfo): Promise<void>;
+    isSyncDone(): boolean;
+    isFileSaved(): boolean;
+    stat(): NonNullable<Stats['sync']>;
 }
 interface ContainerWithDelete<T> {
     delete(key: T): void;
 }
+export declare function readAndCheckStatus(objFolder: string, objId: ObjId): Promise<ObjStatusInfo>;
 export {};