core-3nweb-client-lib 0.27.1 → 0.27.3

package/build/core/storage/index.js

@@ -85,20 +85,14 @@ async function readRootKeyDerivParamsFromCache(folder) {
         throw (0, error_1.errWithCause)(err, `Can't read and parse content of obj status file ${KD_PARAMS_FILE_NAME} in folder ${folder}`);
     }
 }
-/**
- * This function tries to get key derivation parameters from cache on a disk.
- * If not found, it will ask storage server for it with a provided function.
- * @param folder
- * @param getFromServer
- */
-async function getRootKeyDerivParams(folder, getFromServer) {
-    let params = await readRootKeyDerivParamsFromCache(folder);
-    if (!params) {
-        params = await getFromServer();
-        await fs.writeFile((0, path_1.join)(folder, KD_PARAMS_FILE_NAME), JSON.stringify(params), { encoding: 'utf8' });
-    }
+async function getAndCacheRootKeyDerivParamsFromServer(folder, getFromServer) {
+    const params = await getFromServer();
+    await saveRootKeyDerivParamsToDisk(folder, params);
     return params;
 }
+async function saveRootKeyDerivParamsToDisk(folder, params) {
+    await fs.writeFile((0, path_1.join)(folder, KD_PARAMS_FILE_NAME), JSON.stringify(params), { encoding: 'utf8' });
+}
 class StorageAndFS {
     constructor(storage) {
         this.storage = storage;
@@ -125,32 +119,15 @@ class StorageAndFS {
             }
         }
     }
-    static async newOrExisting(storage, key) {
-        var _a, _b;
+    static async makeNew(storage, key) {
+        var _a;
         const s = new StorageAndFS(storage);
-        try {
-            s.rootFS = await fs_1.XspFS.fromExistingRoot(s.storage, key);
-            if ((_a = s.rootFS.v) === null || _a === void 0 ? void 0 : _a.sync) {
-                s.syncedAppDataRoots = await apps_data_1.AppDataFolders.make(s.rootFS);
-            }
-            return s;
-        }
-        catch (err) {
-            if (err.objNotFound) {
-                s.rootFS = await fs_1.XspFS.makeNewRoot(s.storage, key);
-                await (0, system_folders_1.initSysFolders)(s.rootFS);
-                if ((_b = s.rootFS.v) === null || _b === void 0 ? void 0 : _b.sync) {
-                    s.syncedAppDataRoots = await apps_data_1.AppDataFolders.make(s.rootFS);
-                }
-                return s;
-            }
-            else if (err.failedCipherVerification) {
-                return;
-            }
-            else {
-                throw err;
-            }
+        s.rootFS = await fs_1.XspFS.makeNewRoot(s.storage, key);
+        await (0, system_folders_1.initSysFolders)(s.rootFS);
+        if ((_a = s.rootFS.v) === null || _a === void 0 ? void 0 : _a.sync) {
+            s.syncedAppDataRoots = await apps_data_1.AppDataFolders.make(s.rootFS);
         }
+        return s;
     }
     /**
      * This creates app data folder.
@@ -278,30 +255,32 @@ class Storages {
         if (!this.local) {
             return;
         }
-        return async (getSigner) => {
-            if (this.synced) {
-                return true;
-            }
-            const { startObjProcs, syncedStore } = await storage_1.SyncedStore.makeAndStart((0, path_1.join)(storageDir, SYNCED_STORAGE_DIR), user, getSigner, this.storageGetterForSyncedStorage, this.cryptor, () => resolver(user), makeNet(), logError);
-            this.synced = await StorageAndFS.existing(syncedStore, key);
-            key.fill(0);
-            if (!this.synced) {
-                return false;
-            }
-            startObjProcs();
-            return true;
-        };
+        const { syncedStore, setupRemoteAndStartObjProcs } = await storage_1.SyncedStore.makeAndStartWithoutRemote((0, path_1.join)(storageDir, SYNCED_STORAGE_DIR), user, this.storageGetterForSyncedStorage, this.cryptor, () => resolver(user), makeNet(), logError);
+        this.synced = await StorageAndFS.existing(syncedStore, key);
+        key.fill(0);
+        if (!this.synced) {
+            return;
+        }
+        return setupRemoteAndStartObjProcs;
+    }
+    async initFromRemote(user, getSigner, generateKey, makeNet, resolver, logError) {
+        const storageDir = this.storageDirForUser(user);
+        const { startObjProcs, syncedStore } = await storage_1.SyncedStore.makeAndStart((0, path_1.join)(storageDir, SYNCED_STORAGE_DIR), user, getSigner, this.storageGetterForSyncedStorage, this.cryptor, () => resolver(user), makeNet(), logError);
+        const params = await getAndCacheRootKeyDerivParamsFromServer(storageDir, syncedStore.getRootKeyDerivParamsFromServer);
+        const key = await generateKey(params);
+        this.synced = await StorageAndFS.existing(syncedStore, key);
+        this.local = await StorageAndFS.makeNew(await storage_2.LocalStorage.makeAndStart((0, path_1.join)(storageDir, LOCAL_STORAGE_DIR), this.storageGetterForLocalStorage, this.cryptor, logError), key);
+        key.fill(0);
+        startObjProcs();
+        return (!!this.synced && !!this.local);
     }
-    async initFromRemote(user, getSigner, keyOrGen, makeNet, resolver, logError) {
+    async initFreshForNewUser(user, getSigner, params, key, makeNet, resolver, logError) {
         const storageDir = this.storageDirForUser(user);
         const { startObjProcs, syncedStore } = await storage_1.SyncedStore.makeAndStart((0, path_1.join)(storageDir, SYNCED_STORAGE_DIR), user, getSigner, this.storageGetterForSyncedStorage, this.cryptor, () => resolver(user), makeNet(), logError);
-        // getting parameters records them locally on a disk
-        const params = await getRootKeyDerivParams(storageDir, syncedStore.getRootKeyDerivParamsFromServer);
-        const key = ((typeof keyOrGen === 'function') ?
-            await keyOrGen(params) : keyOrGen);
-        this.synced = await StorageAndFS.newOrExisting(syncedStore, key);
-        this.local = await StorageAndFS.newOrExisting(await storage_2.LocalStorage.makeAndStart((0, path_1.join)(storageDir, LOCAL_STORAGE_DIR), this.storageGetterForLocalStorage, this.cryptor, logError), key);
+        this.synced = await StorageAndFS.makeNew(syncedStore, key);
+        this.local = await StorageAndFS.makeNew(await storage_2.LocalStorage.makeAndStart((0, path_1.join)(storageDir, LOCAL_STORAGE_DIR), this.storageGetterForLocalStorage, this.cryptor, logError), key);
         key.fill(0);
+        await saveRootKeyDerivParamsToDisk(storageDir, params);
         startObjProcs();
         return (!!this.synced && !!this.local);
     }
@@ -523,7 +502,7 @@ async function applyPolicyToFSCollection(c, policy, path) {
     }
     const nameAndItem = await (0, async_iter_1.asyncFind)(await c.entries(), async (v) => path.startsWith(v[0]));
     if (!nameAndItem) {
-        throw (0, file_1.makeFileException)(file_1.Code.notFound, path);
+        throw (0, file_1.makeFileException)('notFound', path);
     }
     const [name, item] = nameAndItem;
     path = path.substring(name.length);

package/build/core/storage/local/storage.d.ts

@@ -9,7 +9,7 @@ export declare class LocalStorage implements Storage {
     private readonly files;
     private readonly getStorages;
     readonly cryptor: AsyncSBoxCryptor;
-    private readonly logError;
+    readonly logError: LogError;
     readonly type = "local";
     readonly versioned = true;
     readonly nodes: NodesContainer;

package/build/core/storage/synced/obj-files-gc.d.ts

@@ -1,10 +1,8 @@
 import { SynchronizerOnObjId, SyncedObj } from './obj-files';
-import { StorageOwner as RemoteStorage } from '../../../lib-client/3nstorage/service';
 export declare class GC {
     private readonly sync;
     private readonly rmObjFromCache;
     private readonly rmObjFolder;
-    private readonly remote;
     private isStopped;
     /**
      * All gc steps are done in this process.
@@ -20,11 +18,10 @@ export declare class GC {
      * with wip set.
      */
     private scheduled;
-    constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>, remote: RemoteStorage);
+    constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>);
     scheduleCollection: (obj: SyncedObj) => void;
     stop(): Promise<void>;
     private objCollecting;
     private collectIn;
-    private upsyncObjRemovalWhenNeeded;
     private checkAndRemoveWholeObjFolder;
 }

package/build/core/storage/synced/obj-files-gc.js

@@ -24,11 +24,10 @@ const path_1 = require("path");
 const utils_1 = require("../common/utils");
 const upload_header_file_1 = require("./upload-header-file");
 class GC {
-    constructor(sync, rmObjFromCache, rmObjFolder, remote) {
+    constructor(sync, rmObjFromCache, rmObjFolder) {
         this.sync = sync;
         this.rmObjFromCache = rmObjFromCache;
         this.rmObjFolder = rmObjFolder;
-        this.remote = remote;
         this.isStopped = false;
         /**
          * All gc steps are done in this process.
@@ -71,27 +70,11 @@ class GC {
         this.isStopped = true;
     }
     async collectIn(obj) {
-        if (obj.statusObj().isArchived()) {
-            await this.upsyncObjRemovalWhenNeeded(obj);
-        }
         const nonGarbage = obj.statusObj().getNonGarbageVersions();
         if (!(await this.checkAndRemoveWholeObjFolder(obj, nonGarbage))) {
             await removeGarbageFiles(obj, nonGarbage);
         }
     }
-    async upsyncObjRemovalWhenNeeded(obj) {
-        if (!needsUpsyncOfRemoval(obj)) {
-            return;
-        }
-        try {
-            await this.remote.deleteObj(obj.objId);
-        }
-        catch (exc) {
-            // XXX will need to schedule other attempt to delete on remote
-            return;
-        }
-        await obj.statusObj().recordSyncOfObjRemoval();
-    }
     async checkAndRemoveWholeObjFolder(obj, { local, remote }) {
         if (obj.objId
             && obj.statusObj().isArchived()

package/build/core/storage/synced/obj-files.d.ts

@@ -13,6 +13,12 @@ import { StorageOwner as RemoteStorage } from '../../../lib-client/3nstorage/ser
 import { UploadHeaderChange } from '../../../lib-client/3nstorage/xsp-fs/common';
 export declare const UNSYNCED_FILE_NAME_EXT = "unsynced";
 export declare const REMOTE_FILE_NAME_EXT = "v";
+/**
+ * File system has nodes. Each node may have data in one or many objects of
+ * storage. SyncedObj allows to work with files of storage object, even when
+ * file system node no longer exists. ObjFiles is a holder and factory of
+ * SyncedObj's.
+ */
 export declare class ObjFiles {
     private readonly folders;
     private readonly logError;
@@ -22,6 +28,7 @@ export declare class ObjFiles {
     private readonly gc;
     private constructor();
     static makeFor(path: string, remote: RemoteStorage, logError: LogError): Promise<ObjFiles>;
+    private canMoveObjToDeeperCache;
     findObj(objId: ObjId): Promise<SyncedObj | undefined>;
     getObjInCache(objId: ObjId): SyncedObj | undefined;
     private makeObj;
@@ -31,7 +38,7 @@ export declare class ObjFiles {
         fileWrite$: Observable<FileWrite[]>;
         newObj: SyncedObj;
     }>;
-    findUnsyncedObjs(): Observable<ObjId>;
+    findObjsToRemoveOnRemote(): Observable<ObjId>;
     scheduleGC(obj: SyncedObj): void;
 }
 export declare type SynchronizerOnObjId = <T>(objId: ObjId, action: () => Promise<T>) => Promise<T>;
@@ -81,4 +88,5 @@ export declare class SyncedObj {
     }>;
     syncStatus(): SyncedObjStatus;
     statusObj(): ObjStatus;
+    recordRemovalUploadAndGC(): Promise<void>;
 }

package/build/core/storage/synced/obj-files.js

@@ -19,7 +19,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SyncedObj = exports.ObjFiles = exports.REMOTE_FILE_NAME_EXT = exports.UNSYNCED_FILE_NAME_EXT = void 0;
 const rxjs_1 = require("rxjs");
 const synced_1 = require("../../../lib-common/processes/synced");
-const sleep_1 = require("../../../lib-common/processes/sleep");
 const obj_folders_1 = require("../../../lib-client/objs-on-disk/obj-folders");
 const fs = require("../../../lib-common/async-fs-node");
 const obj_on_disk_1 = require("../../../lib-client/objs-on-disk/obj-on-disk");
@@ -35,6 +34,12 @@ const upload_header_file_1 = require("./upload-header-file");
 const utils_1 = require("../common/utils");
 exports.UNSYNCED_FILE_NAME_EXT = 'unsynced';
 exports.REMOTE_FILE_NAME_EXT = 'v';
+/**
+ * File system has nodes. Each node may have data in one or many objects of
+ * storage. SyncedObj allows to work with files of storage object, even when
+ * file system node no longer exists. ObjFiles is a holder and factory of
+ * SyncedObj's.
+ */
 class ObjFiles {
     constructor(folders, remote, logError) {
         this.folders = folders;
@@ -46,31 +51,32 @@ class ObjFiles {
             if (this.objs.get(obj.objId) === obj) {
                 this.objs.delete(obj.objId);
             }
-        }, objId => this.folders.removeFolderOf(objId), remote);
+        }, objId => this.folders.removeFolderOf(objId));
         Object.freeze(this);
     }
     static async makeFor(path, remote, logError) {
-        const canMove = async (objId, objFolderPath) => {
-            if (objFiles.objs.has(objId)) {
-                return false;
-            }
-            const lst = await fs.readdir(objFolderPath);
-            for (const fName of lst) {
-                if (fName.endsWith(exports.UNSYNCED_FILE_NAME_EXT)) {
-                    return false;
-                }
-            }
-            try {
-                return (await obj_status_1.ObjStatus.fileShowsObjNotInSyncedState(objFolderPath, objId));
-            }
-            catch (exc) {
-                return false;
-            }
-        };
+        const canMove = (objId, objFolderPath) => objFiles.canMoveObjToDeeperCache(objId, objFolderPath);
         const folders = await obj_folders_1.ObjFolders.makeWithGenerations(path, canMove, logError);
         const objFiles = new ObjFiles(folders, remote, logError);
         return objFiles;
     }
+    async canMoveObjToDeeperCache(objId, objFolderPath) {
+        if (this.objs.has(objId)) {
+            return false;
+        }
+        const lst = await fs.readdir(objFolderPath);
+        for (const fName of lst) {
+            if (fName.endsWith(exports.UNSYNCED_FILE_NAME_EXT)) {
+                return false;
+            }
+        }
+        try {
+            return (await obj_status_1.ObjStatus.fileShowsObjNotInSyncedState(objFolderPath, objId));
+        }
+        catch (exc) {
+            return false;
+        }
+    }
     async findObj(objId) {
         let obj = this.objs.get(objId);
         if (obj) {
@@ -131,25 +137,23 @@ class ObjFiles {
             .pipe((0, utils_for_observables_1.flatTap)(undefined, err => this.removeFailedNewObj(newObj)));
         return { fileWrite$, newObj };
     }
-    findUnsyncedObjs() {
+    findObjsToRemoveOnRemote() {
         return (0, rxjs_1.from)([undefined])
             .pipe(
         // listing recent folders, exactly once
         (0, operators_1.mergeMap)(() => this.folders.listRecent()), 
         // flatten array and space it in time, to process folders one by one
-        (0, operators_1.mergeMap)(objsAndPaths => objsAndPaths), (0, operators_1.filter)(({ objId }) => !this.objs.has(objId)), (0, operators_1.mergeMap)(async (objsAndPaths) => {
-            await (0, sleep_1.sleep)(20);
-            return objsAndPaths;
-        }, 1), 
-        // check, emiting objId, if not synced, and undefined, if synced
-        (0, operators_1.mergeMap)(({ path, objId }) => this.sync(objId, async () => {
-            if (this.objs.has(objId)) {
-                return;
+        (0, operators_1.mergeMap)(objsAndPaths => objsAndPaths), (0, operators_1.mergeMap)(async ({ objId, path }) => {
+            const obj = this.objs.get(objId);
+            if (obj) {
+                return (obj.statusObj().needsRemovalOnRemote() ?
+                    objId : undefined);
+            }
+            else {
+                const needsRm = await obj_status_1.ObjStatus.fileShowsObjNeedsRemovalOnRemote(path, objId);
+                return (needsRm ? objId : undefined);
             }
-            const notSynced = await obj_status_1.ObjStatus.fileShowsObjNotInSyncedState(path, objId)
-                .catch(notFoundOrReThrow).catch(this.logError);
-            return (notSynced ? objId : undefined);
-        })), (0, operators_1.filter)(objId => (objId !== undefined)));
+        }, 1), (0, operators_1.filter)(objId => (objId !== undefined)));
     }
     scheduleGC(obj) {
         this.gc.scheduleCollection(obj);
@@ -336,7 +340,7 @@ class SyncedObj {
         else {
             await fs.rename(this.localVerPath(localVersion), remotePath);
         }
-        await this.status.markLocalVersionSynced(localVersion, uploadVersion);
+        await this.status.recordUploadCompletion(localVersion, uploadVersion);
         this.scheduleSelfGC();
     }
     dropCachedLocalObjVersionsLessOrEqual(version) {
@@ -387,6 +391,10 @@ class SyncedObj {
     statusObj() {
         return this.status;
     }
+    async recordRemovalUploadAndGC() {
+        await this.status.recordRemoteRemovalCompletion();
+        this.scheduleSelfGC();
+    }
 }
 exports.SyncedObj = SyncedObj;
 Object.freeze(SyncedObj.prototype);

package/build/core/storage/synced/obj-status.d.ts

@@ -18,11 +18,11 @@ export interface ObjStatusInfo {
     local?: LocalVersions;
     synced?: SyncMarker;
     remote: VersionsOnServer;
+    upload?: UploadInfo;
 }
 export interface LocalVersions extends VersionsInfo {
     isArchived?: boolean;
     archived?: undefined;
-    upload?: UploadInfo;
 }
 export interface SyncMarker {
     version?: number;
@@ -32,12 +32,19 @@ export interface SyncMarker {
 export interface VersionsOnServer extends VersionsInfo {
     isArchived?: boolean;
 }
-export interface UploadInfo {
+export declare type UploadInfo = NewVersionUpload | RemovalUpload;
+export interface NewVersionUpload {
+    type: 'new-version';
     localVersion: number;
     uploadVersion: number;
     baseVersion?: number;
     needUpload?: WholeVerOrderedUpload | DiffVerOrderedUpload;
 }
+export interface RemovalUpload {
+    type: 'removal';
+    isPostponed: boolean;
+    localVersion?: number;
+}
 export interface WholeVerOrderedUpload {
     type: 'ordered-whole';
     createObj?: boolean;
@@ -74,20 +81,24 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
     static makeNew(objFolder: string, objId: ObjId, logError: LogError): Promise<ObjStatus>;
     static makeForDownloadedVersion(objFolder: string, objId: ObjId, version: number, currentRemote: number, logError: LogError): Promise<ObjStatus>;
     static fileShowsObjNotInSyncedState(objFolder: string, objId: ObjId): Promise<boolean>;
+    static fileShowsObjNeedsRemovalOnRemote(objFolder: string, objId: ObjId): Promise<boolean>;
+    needsRemovalOnRemote(): boolean;
+    clearPostponeFlagInRemovalOnRemote(): Promise<boolean>;
     private updateStateIndicator;
     isArchived(): boolean;
     getCurrentLocalOrSynced(): number;
     getNonGarbageVersions(): NonGarbage;
     removeCurrentVersion(): Promise<void>;
     private triggerSaveProc;
-    recordUploadStart(info: UploadInfo): Promise<void>;
-    recordUploadInterimState(info: UploadInfo): Promise<void>;
-    recordUploadCancellation(info: UploadInfo): Promise<void>;
+    recordUploadStart(info: NewVersionUpload): Promise<void>;
+    recordUploadInterimState(info: NewVersionUpload): Promise<void>;
+    recordUploadCancellation(info: NewVersionUpload): Promise<void>;
+    recordUploadCompletion(localVersion: number, uploadVersion: number): Promise<void>;
     recordArchVersionRemoval(version: number): Promise<void>;
     recordVersionArchival(version: number): Promise<void>;
     recordRemoteRemoval(): Promise<void>;
+    recordRemoteRemovalCompletion(): Promise<void>;
     recordRemoteChange(version: number): Promise<void>;
-    recordSyncOfObjRemoval(): Promise<void>;
     recordStatusFromServer({ archived, current }: RemoteObjStatus): Promise<void>;
     /**
      * When given object version is a diff on some base, this method returns
@@ -102,7 +113,6 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
         localBases?: number[];
         syncedBase?: number;
     } | undefined;
-    markLocalVersionSynced(localVersion: number, uploadVersion: number): Promise<void>;
     setLocalCurrentVersion(version: number, baseVer: number | undefined): Promise<void>;
     listVersions(): {
         current?: number;
@@ -113,6 +123,7 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
     latestSyncedVersion(): number | undefined;
     syncStatus(): SyncStatus;
     neverUploaded(): boolean;
+    versionBeforeUnsyncedRemoval(): number | undefined;
     adoptRemoteVersion(version?: number, dropLocalVer?: boolean): Promise<void>;
 }
 export declare function readAndCheckStatus(objFolder: string, objId: ObjId): Promise<ObjStatusInfo>;