core-3nweb-client-lib 0.25.6 → 0.27.0

package/build/core/storage/local/obj-status.js

@@ -17,168 +17,101 @@
 */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObjStatus = void 0;
-const fs = require("../../../lib-common/async-fs-node");
-const processes_1 = require("../../../lib-common/processes");
 const path_1 = require("path");
 const exceptions_1 = require("../../../lib-client/3nstorage/exceptions");
 const obj_info_file_1 = require("../common/obj-info-file");
+const json_saving_1 = require("../common/json-saving");
+const obj_info_file_2 = require("../common/obj-info-file");
+const assert_1 = require("../../../lib-common/assert");
 const STATUS_FILE_NAME = 'status';
 class ObjStatus {
-    constructor(objFolder, status) {
+    constructor(objFolder, status, logError) {
         this.objFolder = objFolder;
         this.status = status;
-        this.saveProc = new processes_1.DeduppedRunner(() => this.saveFile());
+        this.logError = logError;
+        this.saveProc = new json_saving_1.JSONSavingProc((0, path_1.join)(this.objFolder, STATUS_FILE_NAME), () => this.status);
         Object.freeze(this);
     }
-    static async readFrom(objFolder, objId) {
+    static async readFrom(objFolder, objId, logError) {
         const status = await readAndCheckStatus(objFolder, objId);
-        return new ObjStatus(objFolder, status);
+        return new ObjStatus(objFolder, status, logError);
     }
-    static async makeNew(objFolder, objId) {
+    static async makeNew(objFolder, objId, logError) {
         const status = {
             objId,
-            archivedVersions: [],
-            baseToDiff: {},
-            diffToBase: {}
+            versions: {
+                baseToDiff: {},
+                diffToBase: {}
+            }
         };
-        const s = new ObjStatus(objFolder, status);
-        await s.saveProc.trigger();
+        const s = new ObjStatus(objFolder, status, logError);
+        await s.triggerSaveProc();
         return s;
     }
-    async saveFile() {
-        await fs.writeFile(path_1.join(this.objFolder, STATUS_FILE_NAME), JSON.stringify(this.status), { encoding: 'utf8' });
+    triggerSaveProc(captureErrors = false) {
+        let p = this.saveProc.trigger();
+        return (captureErrors ? p.catch(this.logError) : p);
     }
     isArchived() {
         return !!this.status.isArchived;
     }
     getCurrentVersionOrThrow() {
-        if (typeof this.status.currentVersion !== 'number') {
+        if (typeof this.status.versions.current !== 'number') {
             throw new Error(`Object ${this.status.objId} has no current version.`);
         }
-        return this.status.currentVersion;
+        return this.status.versions.current;
     }
     getNonGarbageVersions() {
         return {
-            nonGarbage: nonGarbageVersions(this.status),
-            gcMaxVer: this.status.currentVersion
+            nonGarbage: (0, obj_info_file_2.nonGarbageVersionsIn)(this.status.versions),
+            gcMaxVer: this.status.versions.current
         };
     }
     async setNewCurrentVersion(newVersion, baseVer) {
-        if (baseVer !== undefined) {
-            // base->diff links should be added before removals
-            addBaseToDiffLinkInStatus(this.status, newVersion, baseVer);
-        }
-        if (typeof this.status.currentVersion === 'number') {
-            rmNonArchVersionsIn(this.status, this.status.currentVersion);
-        }
-        this.status.currentVersion = newVersion;
-        await this.saveProc.trigger();
+        (0, obj_info_file_2.setCurrentVersionIn)(this.status.versions, newVersion, baseVer);
+        await this.triggerSaveProc();
     }
     async removeCurrentVersion(verObjs) {
         this.status.isArchived = true;
-        if (typeof this.status.currentVersion === 'number') {
-            verObjs.delete(this.status.currentVersion);
-            rmNonArchVersionsIn(this.status, this.status.currentVersion);
-            delete this.status.currentVersion;
+        const current = (0, obj_info_file_2.rmCurrentVersionIn)(this.status.versions);
+        if (typeof current === 'number') {
+            verObjs.delete(current);
         }
-        await this.saveProc.trigger();
+        await this.triggerSaveProc();
     }
     async removeArchivedVersion(version, verObjs) {
         verObjs.delete(version);
-        const arch = this.status.archivedVersions;
-        const vInd = arch.indexOf(version);
-        if (vInd < 0) {
-            return;
-        }
-        arch.splice(vInd, 1);
-        rmNonArchVersionsIn(this.status, version);
-        await this.saveProc.trigger();
+        (0, obj_info_file_2.rmArchVersionFrom)(this.status.versions, version);
+        await this.triggerSaveProc();
+    }
+    async archiveCurrentVersion() {
+        const versions = this.status.versions;
+        (0, assert_1.assert)(!!versions.current);
+        (0, obj_info_file_1.addArchived)(versions, versions.current);
+        await this.triggerSaveProc();
+    }
+    listVersions() {
+        const archived = this.status.versions.archived;
+        return {
+            current: this.status.versions.current,
+            archived: (archived ? archived.slice() : [])
+        };
     }
 }
 exports.ObjStatus = ObjStatus;
 Object.freeze(ObjStatus.prototype);
 Object.freeze(ObjStatus);
-/**
- * This function adds base->diff link to status. Status object is changed in
- * this call.
- * @param status into which a link between versions should be added
- * @param diffVer
- * @param baseVer
- */
-function addBaseToDiffLinkInStatus(status, diffVer, baseVer) {
-    if (diffVer <= baseVer) {
-        throw new Error(`Given diff version ${diffVer} is not greater than base version ${baseVer}`);
-    }
-    status.diffToBase[diffVer] = baseVer;
-    const diffs = status.baseToDiff[baseVer];
-    if (diffs) {
-        if (diffs.indexOf(diffVer) < 0) {
-            diffs.push(diffVer);
-        }
-    }
-    else {
-        status.baseToDiff[baseVer] = [diffVer];
-    }
-}
-/**
-* This function removes given version from status object, if it is neither
-* archived, nor is a base for another version. If given version is itself
-* based on another, this function is recursively applied to base version, as
-* well.
-* @param status in which version(s) should be removed
-* @param ver
-*/
-function rmNonArchVersionsIn(status, ver) {
-    if (status.archivedVersions.indexOf(ver) >= 0) {
-        return;
-    }
-    if (status.baseToDiff[ver]) {
-        return;
-    }
-    const base = status.diffToBase[ver];
-    if (typeof base !== 'number') {
-        return;
-    }
-    delete status.diffToBase[ver];
-    const diffs = status.baseToDiff[base];
-    if (!diffs) {
-        return;
-    }
-    const diffInd = diffs.indexOf(ver);
-    if (diffInd < 0) {
-        return;
-    }
-    diffs.splice(diffInd, 1);
-    if (diffs.length === 0) {
-        delete status.baseToDiff[base];
-        rmNonArchVersionsIn(status, base);
-    }
-}
 async function readAndCheckStatus(objFolder, objId) {
-    const status = await obj_info_file_1.readJSONInfoFileIn(objFolder, STATUS_FILE_NAME);
+    const status = await (0, obj_info_file_2.readJSONInfoFileIn)(objFolder, STATUS_FILE_NAME);
     if (!status) {
-        throw exceptions_1.makeStorageException({
+        throw (0, exceptions_1.makeStorageException)({
             message: `Obj status file is not found in obj folder ${objFolder}`
         });
     }
     // XXX we may do some checks and sanitization here
     if (objId !== status.objId) {
-        throw exceptions_1.makeStorageException({ message: `Invalid objId in status file for obj ${objId}, in folder ${objFolder}.\nInvalid content:\n${JSON.stringify(status, null, 2)}` });
+        throw (0, exceptions_1.makeStorageException)({ message: `Invalid objId in status file for obj ${objId}, in folder ${objFolder}.\nInvalid content:\n${JSON.stringify(status, null, 2)}` });
     }
     return status;
 }
-function nonGarbageVersions(status) {
-    const nonGarbage = new Set();
-    addWithBasesTo(nonGarbage, status.currentVersion, status);
-    for (const archVer of status.archivedVersions) {
-        addWithBasesTo(nonGarbage, archVer, status);
-    }
-    return nonGarbage;
-}
-function addWithBasesTo(nonGarbage, ver, status) {
-    while (typeof ver === 'number') {
-        nonGarbage.add(ver);
-        ver = status.diffToBase[ver];
-    }
-}
 Object.freeze(exports);

package/build/core/storage/local/storage.d.ts

@@ -1,21 +1,31 @@
-import { Storage, NodesContainer, StorageGetter, ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
+import { Storage, NodesContainer, StorageGetter, ObjId, NodeEvent, LocalObjStatus } from '../../../lib-client/3nstorage/xsp-fs/common';
 import { LogError } from '../../../lib-client/logging/log-to-file';
 import { AsyncSBoxCryptor, Subscribe, ObjSource } from 'xsp-files';
+import { Observable } from 'rxjs';
+declare type FolderEvent = web3n.files.FolderEvent;
+declare type FileEvent = web3n.files.FileEvent;
+declare type FSType = web3n.files.FSType;
 export declare class LocalStorage implements Storage {
     private readonly files;
     private readonly getStorages;
     readonly cryptor: AsyncSBoxCryptor;
     private readonly logError;
-    readonly type: web3n.files.FSType;
+    readonly type = "local";
     readonly versioned = true;
     readonly nodes: NodesContainer;
+    readonly connect: undefined;
+    private readonly events;
     private constructor();
     static makeAndStart(path: string, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, logError: LogError): Promise<Storage>;
-    storageForLinking(type: web3n.files.FSType, location?: string): Storage;
+    getNodeEvents(): Observable<NodeEvent>;
+    broadcastNodeEvent(objId: ObjId, parentObjId: ObjId | undefined, childObjId: ObjId | undefined, event: FolderEvent | FileEvent): void;
+    storageForLinking(type: FSType, location?: string): Storage;
+    private getObjOrThrow;
+    status(objId: ObjId): Promise<LocalObjStatus>;
     generateNewObjId(): Promise<string>;
-    private getObjNonArchOrThrow;
-    getObj(objId: ObjId): Promise<ObjSource>;
+    getObjSrc(objId: ObjId, version?: number): Promise<ObjSource>;
     saveObj(objId: string, version: number, encSub: Subscribe): Promise<void>;
     removeObj(objId: string): Promise<void>;
     close(): Promise<void>;
 }
+export {};

package/build/core/storage/local/storage.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2016 - 2020 3NSoft Inc.
+ Copyright (C) 2016 - 2020, 2022 3NSoft Inc.
  
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -13,7 +13,8 @@
  See the GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalStorage = void 0;
 const common_1 = require("../../../lib-client/3nstorage/xsp-fs/common");
@@ -22,6 +23,7 @@ const random_node_1 = require("../../../lib-common/random-node");
 const ecma_nacl_1 = require("ecma-nacl");
 const buffer_utils_1 = require("../../../lib-common/buffer-utils");
 const obj_files_1 = require("./obj-files");
+const utils_for_observables_1 = require("../../../lib-common/utils-for-observables");
 class LocalStorage {
     constructor(files, getStorages, cryptor, logError) {
         this.files = files;
@@ -31,12 +33,20 @@ class LocalStorage {
         this.type = 'local';
         this.versioned = true;
         this.nodes = new common_1.NodesContainer();
+        this.connect = undefined;
+        this.events = new utils_for_observables_1.Broadcast();
         Object.seal(this);
     }
     static async makeAndStart(path, getStorages, cryptor, logError) {
         const files = await obj_files_1.ObjFiles.makeFor(path, logError);
         const s = new LocalStorage(files, getStorages, cryptor, logError);
-        return common_1.wrapStorageImplementation(s);
+        return (0, common_1.wrapStorageImplementation)(s);
+    }
+    getNodeEvents() {
+        return this.events.event$;
+    }
+    broadcastNodeEvent(objId, parentObjId, childObjId, event) {
+        this.events.next({ objId, parentObjId, childObjId, event });
     }
     storageForLinking(type, location) {
         if ((type === 'local') || (type === 'synced')) {
@@ -49,8 +59,22 @@ class LocalStorage {
             throw new Error(`Getting ${type} storage is not implemented in local storage.`);
         }
     }
+    async getObjOrThrow(objId, allowArchived = false) {
+        const obj = await this.files.findObj(objId);
+        if (!obj
+            || (!allowArchived && obj.statusObj().isArchived())) {
+            throw (0, exceptions_1.makeObjNotFoundExc)(objId);
+        }
+        else {
+            return obj;
+        }
+    }
+    async status(objId) {
+        const obj = await this.getObjOrThrow(objId);
+        return obj.localStatus();
+    }
     async generateNewObjId() {
-        const nonce = await random_node_1.bytes(ecma_nacl_1.secret_box.NONCE_LENGTH);
+        const nonce = await (0, random_node_1.bytes)(ecma_nacl_1.secret_box.NONCE_LENGTH);
         const id = buffer_utils_1.base64urlSafe.pack(nonce);
         if (this.nodes.reserveId(id)) {
             return id;
@@ -59,40 +83,35 @@ class LocalStorage {
             return this.generateNewObjId();
         }
     }
-    async getObjNonArchOrThrow(objId) {
-        const obj = await this.files.findObj(objId);
-        if (!obj || obj.isArchived()) {
-            throw exceptions_1.makeObjNotFoundExc(objId);
+    async getObjSrc(objId, version) {
+        const obj = await this.getObjOrThrow(objId);
+        if (!version) {
+            version = obj.statusObj().getCurrentVersionOrThrow();
         }
-        return obj;
-    }
-    async getObj(objId) {
-        const obj = await this.getObjNonArchOrThrow(objId);
-        const currentVer = obj.getCurrentVersionOrThrow();
-        return obj.getObjSrc(currentVer);
+        return obj.getObjSrc(version);
     }
     async saveObj(objId, version, encSub) {
         const obj = await this.files.findObj(objId);
         if (version === 1) {
             if (obj) {
-                throw exceptions_1.makeObjExistsExc(objId);
+                throw (0, exceptions_1.makeObjExistsExc)(objId);
             }
             await this.files.saveFirstVersion(objId, encSub);
         }
         else {
             if (!obj) {
-                throw exceptions_1.makeObjNotFoundExc(objId);
+                throw (0, exceptions_1.makeObjNotFoundExc)(objId);
             }
             await obj.saveNewVersion(version, encSub);
         }
     }
     async removeObj(objId) {
-        const obj = await this.getObjNonArchOrThrow(objId);
+        const obj = await this.getObjOrThrow(objId);
         await obj.removeCurrentVersion();
     }
     async close() {
         try {
-            // XXX add cleanups
+            this.events.done();
         }
         catch (err) {
             await this.logError(err);

package/build/core/storage/synced/downloader.js

@@ -13,10 +13,11 @@
  See the GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Downloader = void 0;
-const processes_1 = require("../../../lib-common/processes");
+const synced_1 = require("../../../lib-common/processes/synced");
 const buffer_utils_1 = require("../../../lib-common/buffer-utils");
 const MAX_GETTING_CHUNK = 512 * 1024;
 const DOWNLOAD_START_CHUNK = 128 * 1024;
@@ -28,7 +29,7 @@ class Downloader {
          * When it comes to the download start, if chain exists, it means that
          * process has already started.
          */
-        this.downloadProcs = new processes_1.NamedProcs();
+        this.downloadProcs = new synced_1.NamedProcs();
         Object.seal(this);
     }
     async sync(objId, version, action) {
@@ -63,12 +64,12 @@ class Downloader {
                     chunks.push(chunk);
                     ofs += chunk.length;
                 }
-                return buffer_utils_1.joinByteArrs(chunks);
+                return (0, buffer_utils_1.joinByteArrs)(chunks);
             }
         });
     }
-    getCurrentObjVersion(objId) {
-        return this.sync(objId, -1, async () => {
+    async getCurrentObjVersion(objId) {
+        return await this.sync(objId, -1, async () => {
             const { header, segsTotalLen, version, segsChunk } = await this.remoteStorage.getCurrentObj(objId, DOWNLOAD_START_CHUNK);
             const layout = {
                 sections: [{ src: 'new', ofs: 0, len: segsTotalLen }]

package/build/core/storage/synced/obj-files-gc.d.ts

@@ -1,8 +1,10 @@
 import { SynchronizerOnObjId, SyncedObj } from './obj-files';
+import { StorageOwner as RemoteStorage } from '../../../lib-client/3nstorage/service';
 export declare class GC {
     private readonly sync;
     private readonly rmObjFromCache;
     private readonly rmObjFolder;
+    private readonly remote;
     private isStopped;
     /**
      * All gc steps are done in this process.
@@ -18,8 +20,11 @@ export declare class GC {
      * with wip set.
      */
     private scheduled;
-    constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>);
+    constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>, remote: RemoteStorage);
     scheduleCollection: (obj: SyncedObj) => void;
     stop(): Promise<void>;
     private objCollecting;
+    private collectIn;
+    private upsyncObjRemovalWhenNeeded;
+    private checkAndRemoveWholeObjFolder;
 }

package/build/core/storage/synced/obj-files-gc.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2020 3NSoft Inc.
+ Copyright (C) 2020, 2022 3NSoft Inc.
 
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -13,20 +13,27 @@
  See the GNU General Public License for more details.
 
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GC = void 0;
-const processes_1 = require("../../../lib-common/processes");
+const obj_files_1 = require("./obj-files");
+const synced_1 = require("../../../lib-common/processes/synced");
+const fs = require("../../../lib-common/async-fs-node");
+const path_1 = require("path");
+const utils_1 = require("../common/utils");
+const upload_header_file_1 = require("./upload-header-file");
 class GC {
-    constructor(sync, rmObjFromCache, rmObjFolder) {
+    constructor(sync, rmObjFromCache, rmObjFolder, remote) {
         this.sync = sync;
         this.rmObjFromCache = rmObjFromCache;
         this.rmObjFolder = rmObjFolder;
+        this.remote = remote;
         this.isStopped = false;
         /**
          * All gc steps are done in this process.
          */
-        this.gcProc = new processes_1.SingleProc();
+        this.gcProc = new synced_1.SingleProc();
         /**
          * wip are objects that are currently processed. When wip set is empty,
          * it gets swapped with non-empty scheduled set.
@@ -51,11 +58,11 @@ class GC {
             if (this.wip.size === 0) {
                 [this.wip, this.scheduled] = [this.scheduled, this.wip];
             }
-            const obj = getAndRemoveOneFrom(this.wip);
+            const obj = (0, utils_1.getAndRemoveOneFrom)(this.wip);
             if (!obj) {
                 return;
             }
-            // XXX
+            await this.sync(obj.objId, () => this.collectIn(obj).catch(utils_1.noop));
             return this.objCollecting();
         };
         Object.seal(this);
@@ -63,17 +70,103 @@ class GC {
     async stop() {
         this.isStopped = true;
     }
+    async collectIn(obj) {
+        if (obj.statusObj().isArchived()) {
+            await this.upsyncObjRemovalWhenNeeded(obj);
+        }
+        const nonGarbage = obj.statusObj().getNonGarbageVersions();
+        if (!(await this.checkAndRemoveWholeObjFolder(obj, nonGarbage))) {
+            await removeGarbageFiles(obj, nonGarbage);
+        }
+    }
+    async upsyncObjRemovalWhenNeeded(obj) {
+        if (!needsUpsyncOfRemoval(obj)) {
+            return;
+        }
+        try {
+            await this.remote.deleteObj(obj.objId);
+        }
+        catch (exc) {
+            // XXX will need to schedule other attempt to delete on remote
+            return;
+        }
+        await obj.statusObj().recordSyncOfObjRemoval();
+    }
+    async checkAndRemoveWholeObjFolder(obj, { local, remote }) {
+        if (obj.objId
+            && obj.statusObj().isArchived()
+            && !needsUpsyncOfRemoval(obj)
+            && (!local || (local.nonGarbage.size === 0))
+            && (remote.nonGarbage.size === 0)) {
+            this.rmObjFromCache(obj);
+            await this.rmObjFolder(obj.objId);
+            return true;
+        }
+        else {
+            return false;
+        }
+    }
 }
 exports.GC = GC;
 Object.freeze(GC.prototype);
 Object.freeze(GC);
-function getAndRemoveOneFrom(set) {
-    const iter = set.values();
-    const { value, done } = iter.next();
-    if (done) {
+function needsUpsyncOfRemoval(obj) {
+    const s = obj.statusObj();
+    return !(s.neverUploaded() || (s.syncStatus().state === 'synced'));
+}
+function canRemove(f, { local, remote, uploadVersion }) {
+    if (f.endsWith(obj_files_1.UNSYNCED_FILE_NAME_EXT)) {
+        const verStr = f.slice(0, f.length - 1 - obj_files_1.UNSYNCED_FILE_NAME_EXT.length);
+        return (local ? canGC(verStr, local) : true);
+    }
+    else if (f.endsWith(obj_files_1.REMOTE_FILE_NAME_EXT)) {
+        const verStr = f.slice(0, f.length - 1 - obj_files_1.REMOTE_FILE_NAME_EXT.length);
+        return canGC(verStr, remote);
+    }
+    else if (!!uploadVersion
+        && f.endsWith(upload_header_file_1.UPLOAD_HEADER_FILE_NAME_EXT)) {
+        const verStr = f.slice(0, f.length - 1 - upload_header_file_1.UPLOAD_HEADER_FILE_NAME_EXT.length);
+        return canGCUploadHeader(verStr, uploadVersion);
+    }
+    else {
+        return false;
+    }
+}
+function canGC(verStr, nonGC) {
+    const ver = parseInt(verStr);
+    if (isNaN(ver)) {
+        return true;
+    }
+    else if (!nonGC.nonGarbage.has(ver)
+        && (!nonGC.gcMaxVer || (ver < nonGC.gcMaxVer))) {
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+function canGCUploadHeader(verStr, uploadVersion) {
+    const ver = parseInt(verStr);
+    if (isNaN(ver)) {
+        return true;
+    }
+    else {
+        return (ver === uploadVersion);
+    }
+}
+async function removeGarbageFiles(obj, nonGarbage) {
+    const lst = await fs.readdir(obj.objFolder).catch(utils_1.noop);
+    if (!lst) {
         return;
     }
-    set.delete(value);
-    return value;
+    const rmProcs = [];
+    for (const f of lst) {
+        if (canRemove(f, nonGarbage)) {
+            rmProcs.push(fs.unlink((0, path_1.join)(obj.objFolder, f)).catch(utils_1.noop));
+        }
+    }
+    if (rmProcs.length > 0) {
+        await Promise.all(rmProcs);
+    }
 }
 Object.freeze(exports);