core-3nweb-client-lib 0.25.3 → 0.26.0

package/build/core/storage/synced/storage.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2015 - 2020 3NSoft Inc.
+ Copyright (C) 2015 - 2020, 2022 3NSoft Inc.
  
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -13,7 +13,8 @@
  See the GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SyncedStore = void 0;
 const common_1 = require("../../../lib-client/3nstorage/xsp-fs/common");
@@ -26,6 +27,7 @@ const xsp_files_1 = require("xsp-files");
 const downloader_1 = require("./downloader");
 const remote_events_1 = require("./remote-events");
 const upsyncer_1 = require("./upsyncer");
+const utils_for_observables_1 = require("../../../lib-common/utils-for-observables");
 class SyncedStore {
     constructor(files, remoteStorage, getStorages, cryptor, logError) {
         this.files = files;
@@ -36,15 +38,17 @@ class SyncedStore {
         this.type = 'synced';
         this.versioned = true;
         this.nodes = new common_1.NodesContainer();
+        this.events = new utils_for_observables_1.Broadcast();
         const getFSNodes = (objId) => this.nodes.get(objId);
         this.remoteEvents = new remote_events_1.RemoteEvents(this.remoteStorage, this.files, getFSNodes, this.logError);
-        this.uploader = new upsyncer_1.UpSyncer(this.remoteStorage, this.files, this.logError);
+        this.uploader = new upsyncer_1.UpSyncer(this.remoteStorage, this.logError, this.broadcastUpSyncEvent.bind(this));
         Object.seal(this);
     }
     static async makeAndStart(path, user, getSigner, getStorages, cryptor, remoteServiceUrl, makeNet, logError) {
         const remote = new service_1.StorageOwner(user, getSigner, remoteServiceUrl, makeNet());
         const objFiles = await obj_files_1.ObjFiles.makeFor(path, new downloader_1.Downloader(remote), logError);
         const s = new SyncedStore(objFiles, remote, getStorages, cryptor, logError);
+        s.uploader.start();
         return {
             syncedStore: common_1.wrapSyncStorageImplementation(s),
             startObjProcs: () => {
@@ -52,6 +56,19 @@ class SyncedStore {
             }
         };
     }
+    getNodeEvents() {
+        return this.events.event$;
+    }
+    broadcastNodeEvent(objId, parentObjId, event) {
+        this.events.next({ objId, parentObjId, event });
+    }
+    broadcastUpSyncEvent(objId, task) {
+        const node = this.nodes.get(objId);
+        if (!node) {
+            return;
+        }
+        node.broadcastUpSyncEvent(task);
+    }
     storageForLinking(type, location) {
         if (type === 'synced') {
             return common_1.wrapStorageImplementation(this);
@@ -63,6 +80,13 @@ class SyncedStore {
             throw new Error(`Getting ${type} storage is not implemented in local storage.`);
         }
     }
+    async getObjSyncInfo(objId) {
+        const obj = await this.files.findObj(objId);
+        if (!obj) {
+            return;
+        }
+        return obj.sync().stat();
+    }
     getRootKeyDerivParamsFromServer() {
         return this.remoteStorage.getKeyDerivParams();
     }
@@ -132,6 +156,7 @@ class SyncedStore {
     async close() {
         try {
             await this.uploader.stop();
+            this.events.done();
             await this.remoteEvents.close();
             await this.remoteStorage.logout();
         }

package/build/core/storage/synced/upsyncer.d.ts

@@ -1,21 +1,22 @@
 import { StorageOwner } from "../../../lib-client/3nstorage/service";
-import { SyncedObj, ObjFiles } from "./obj-files";
+import { SyncedObj } from "./obj-files";
 import { MonoTypeOperatorFunction } from "rxjs";
 import { FileWrite } from "../../../lib-client/objs-on-disk/file-writing-proc";
 import { LogError } from "../../../lib-client/logging/log-to-file";
+import { UpSyncTaskInfo } from "./obj-status";
+import { ObjId } from "../../../lib-client/3nstorage/xsp-fs/common";
+export declare type FileWriteTapOperator = MonoTypeOperatorFunction<FileWrite[]>;
+export declare type BroadcastUpSyncEvent = (objId: ObjId, task: UpSyncTaskInfo) => void;
 export declare class UpSyncer {
     private readonly remoteStorage;
-    private readonly files;
     private readonly logError;
+    private readonly broadcastUpSyncEvent;
+    private readonly execPools;
     private readonly uploads;
-    constructor(remoteStorage: StorageOwner, files: ObjFiles, logError: LogError);
+    constructor(remoteStorage: StorageOwner, logError: LogError, broadcastUpSyncEvent: BroadcastUpSyncEvent);
     private getOrMakeUploadsFor;
     start(): void;
     stop(): Promise<void>;
-    get chunkSize(): number;
-    private readonly addToWorker;
     tapFileWrite(obj: SyncedObj, isNew: boolean, newVersion: number, baseVersion?: number): FileWriteTapOperator;
     removeCurrentVersionOf(obj: SyncedObj): Promise<void>;
-    private readonly worker;
 }
-export declare type FileWriteTapOperator = MonoTypeOperatorFunction<FileWrite[]>;

package/build/core/storage/synced/upsyncer.js

@@ -1,6 +1,6 @@
 "use strict";
 /*
- Copyright (C) 2020 3NSoft Inc.
+ Copyright (C) 2020, 2022 3NSoft Inc.
  
  This program is free software: you can redistribute it and/or modify it under
  the terms of the GNU General Public License as published by the Free Software
@@ -13,139 +13,230 @@
  See the GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License along with
- this program. If not, see <http://www.gnu.org/licenses/>. */
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.UpSyncer = void 0;
 const operators_1 = require("rxjs/operators");
-const processes_1 = require("../../../lib-common/processes");
-const upsync_status_1 = require("./upsync-status");
+const labelled_exec_pools_1 = require("../../../lib-common/processes/labelled-exec-pools");
 const assert_1 = require("../../../lib-common/assert");
 const MAX_CHUNK_SIZE = 512 * 1024;
+const MAX_FAST_UPLOAD = 2 * 1024 * 1024;
+function executorLabelFor(info) {
+    let uploadSize = 0;
+    if (info.header) {
+        uploadSize += info.header;
+    }
+    for (const seg of info.segs) {
+        uploadSize += seg.len;
+    }
+    return ((uploadSize <= MAX_FAST_UPLOAD) ? 'fast' : 'long');
+}
+function addEventToInfo(info, writeEv) {
+    for (const w of writeEv) {
+        if (w.isHeader) {
+            info.header = w.bytes.length;
+        }
+        else {
+            mergeSections(info.segs, w);
+        }
+    }
+}
+function mergeSections(segs, w) {
+    const wStart = w.ofs;
+    const wLen = w.bytes.length;
+    for (let i = 0; i < segs.length; i += 1) {
+        const section = segs[i];
+        const sectionEnd = section.ofs + section.len;
+        if (sectionEnd < wStart) {
+            continue;
+        }
+        if (sectionEnd === wStart) {
+            section.len += wLen;
+        }
+        else if (section.ofs <= wStart) {
+            const maxOverlap = section.len - (wStart - section.ofs);
+            if (maxOverlap < wLen) {
+                section.len += wLen - maxOverlap;
+            }
+        }
+        else if ((wStart + wLen) === section.ofs) {
+            section.ofs = wStart;
+        }
+        else {
+            segs.splice(i, 0, { ofs: wStart, len: wLen });
+        }
+        return;
+    }
+    segs.push({ ofs: wStart, len: wLen });
+}
 class UpSyncer {
-    constructor(remoteStorage, files, logError) {
+    constructor(remoteStorage, logError, broadcastUpSyncEvent) {
         this.remoteStorage = remoteStorage;
-        this.files = files;
         this.logError = logError;
+        this.broadcastUpSyncEvent = broadcastUpSyncEvent;
         this.uploads = new Map();
-        this.addToWorker = (ready) => {
-            this.worker.add(ready);
-        };
-        this.worker = new processes_1.Worker(async (u) => {
-            await u.process();
-            if (u.isDone() && (u === this.uploads.get(u.obj))) {
-                this.uploads.delete(u.obj);
-            }
-        }, async (queue) => {
-            for (const u of queue) {
-                u.stop();
-                this.uploads.delete(u.obj);
-            }
-        });
+        this.execPools = new labelled_exec_pools_1.LabelledExecPools([
+            { label: 'long', maxProcs: 1 },
+            { label: 'fast', maxProcs: 1 }
+        ], logError);
         Object.seal(this);
     }
     getOrMakeUploadsFor(obj) {
         let uploads = this.uploads.get(obj);
         if (!uploads) {
-            const status = new upsync_status_1.UpSyncTasks(obj.objFolder, this.logError);
-            uploads = new ObjUpSync(obj, status, this.remoteStorage, this.addToWorker, this.logError);
+            uploads = new ObjUpSync(obj, this.remoteStorage, this.execPools, this.logError, this.broadcastUpSyncEvent);
             this.uploads.set(obj, uploads);
         }
         return uploads;
     }
     start() {
-        this.worker.start(2);
+        this.execPools.start();
     }
     async stop() {
-        await this.worker.stop();
-        for (const upload of this.uploads.values()) {
-            upload.stop();
-        }
+        await this.execPools.stop(); // implicitly cancels all upsync tasks
         this.uploads.clear();
     }
-    get chunkSize() {
-        return (this.remoteStorage.maxChunkSize ?
-            Math.min(this.remoteStorage.maxChunkSize, MAX_CHUNK_SIZE) :
-            MAX_CHUNK_SIZE);
-    }
     tapFileWrite(obj, isNew, newVersion, baseVersion) {
         const objUploads = this.getOrMakeUploadsFor(obj);
         return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
     }
     async removeCurrentVersionOf(obj) {
         const objUploads = this.getOrMakeUploadsFor(obj);
-        objUploads.removeCurrentVersion();
+        if (objUploads.neededExecutor()) {
+            this.execPools.add(objUploads);
+        }
     }
 }
 exports.UpSyncer = UpSyncer;
 Object.freeze(UpSyncer.prototype);
 Object.freeze(UpSyncer);
 class ObjUpSync {
-    constructor(obj, status, remoteStorage, addToWorker, logError) {
+    constructor(obj, remoteStorage, execPools, logError, broadcastUpSyncEvent) {
         this.obj = obj;
-        this.status = status;
         this.remoteStorage = remoteStorage;
-        this.addToWorker = addToWorker;
+        this.execPools = execPools;
         this.logError = logError;
-        this.objWriteTaps = new Map();
-        Object.freeze(this);
+        this.broadcastUpSyncEvent = broadcastUpSyncEvent;
+        this.taskInProcess = undefined;
+        this.isCancelled = false;
+        this.runningProc = undefined;
+        Object.seal(this);
     }
     tapFileWrite(isObjNew, newVersion, baseVersion) {
-        const tap = new ObjWriteTap(this.obj, newVersion, baseVersion, isObjNew);
-        const uploadInfo = upsync_status_1.makeUploadInfo(tap.version, tap.baseVersion);
-        this.objWriteTaps.set(uploadInfo, tap);
-        return tap.tapOperator();
+        // XXX for now upload is started when write is complete, but tapped code
+        //     layout can adopt upload as before completion.
+        const uploadInfo = {
+            type: 'upload',
+            version: newVersion,
+            baseVersion,
+            needUpload: {
+                segs: []
+            }
+        };
+        if (isObjNew) {
+            uploadInfo.createObj = true;
+        }
+        return operators_1.tap(writeEv => {
+            addEventToInfo(uploadInfo.needUpload, writeEv);
+        }, async (err) => {
+            // XXX cancel in-tap upload, when it will be added.
+            //     Something should be done in case there is a transaction that
+            //     needs  cancelling.
+            await this.cancelTransactionIfOpen(uploadInfo);
+            await this.logError(err);
+        }, () => {
+            uploadInfo.needUpload.allByteOnDisk = true;
+            this.enqueueTask(uploadInfo);
+        });
+    }
+    enqueueTask(task) {
+        this.obj.sync().queueTask(task);
+        if (this.neededExecutor()) {
+            this.execPools.add(this);
+        }
+        else {
+            this.obj.scheduleSelfGC();
+        }
     }
-    stop() {
-        // XXX
-        // - is something needed here?
+    async recordTaskCompletion(task) {
+        this.taskInProcess = undefined;
+        if (task.type === 'upload') {
+            await this.obj.markVersionSynced(task.version);
+        }
+        await this.obj.sync().recordTaskCompletion(task);
+        this.broadcastUpSyncEvent(this.obj.objId, task);
     }
-    removeCurrentVersion() {
-        this.status.queueTask({
-            type: 'removal',
-            currentVersion: true
-        });
-        this.addToWorker(this);
+    neededExecutor() {
+        const task = this.obj.sync().glanceOnNextTask();
+        if (!task) {
+            return;
+        }
+        if (task.type !== 'upload') {
+            return 'fast';
+        }
+        if (!task.needUpload) {
+            return;
+        }
+        return executorLabelFor(task.needUpload);
     }
     removeArchivedVersion(version) {
-        this.status.queueTask({
+        this.enqueueTask({
             type: 'removal',
             archivedVersions: version
         });
-        this.addToWorker(this);
     }
-    isDone() {
-        return this.status.isDone();
+    async cancel() {
+        var _a;
+        this.isCancelled = true;
+        await ((_a = this.runningProc) === null || _a === void 0 ? void 0 : _a.catch(noop));
     }
     async process() {
-        if (this.isDone()) {
+        if (this.obj.sync().isSyncDone() || this.isCancelled) {
+            return;
+        }
+        let task = undefined;
+        try {
+            if (!this.taskInProcess) {
+                this.taskInProcess = (await this.obj.sync().getTaskForProcessing());
+                if (this.isCancelled) {
+                    return;
+                }
+            }
+            task = this.taskInProcess;
+        }
+        catch (err) {
+            await this.logError(err, `ObjUpSync.process fails to get task`);
             return;
         }
         try {
-            const task = (await this.status.nextTask());
             if (task.type === 'upload') {
-                await this.processUpload(task);
+                this.runningProc = this.processUpload(task);
             }
             else if (task.type === 'removal') {
-                await this.processRemoval(task);
+                this.runningProc = this.processRemoval(task);
             }
             else if (task.type === 'archiving') {
-                await this.processArchival(task);
+                this.runningProc = this.processArchival(task);
             }
             else {
                 throw new Error(`This shouldn't be reached`);
             }
+            try {
+                await this.runningProc;
+            }
+            finally {
+                this.runningProc = undefined;
+            }
         }
         catch (err) {
-            await this.logError(err, `Error occured in uploader processing`);
-        }
-        if (!this.isDone()) {
-            this.addToWorker(this);
+            await this.logError(err, `From ObjUpSync.process task ${task.type}`);
         }
     }
     async processArchival(task) {
         // XXX tell server to archive current version
         await this.logError(`Archival of current version is not implemented, yet.`);
-        await this.status.recordTaskCompletion(task);
+        await this.recordTaskCompletion(task);
     }
     async processRemoval(task) {
         if (task.currentVersion) {
@@ -160,33 +251,22 @@ class ObjUpSync {
             // XXX tell server to remove given archived versions
             await this.logError(`Removal of archived version is not implemented, yet.`);
         }
-        await this.status.recordTaskCompletion(task);
+        await this.recordTaskCompletion(task);
+        this.obj.scheduleSelfGC();
     }
     async processUpload(task) {
-        const tap = this.objWriteTaps.get(task);
-        if (tap && !task.done) {
-            await this.uploadReadyBytes(task, tap);
-        }
-        if (tap && !task.done) {
-            await this.status.recordInterimStateOfCurrentTask(task);
+        if (task.needUpload) {
+            if (task.needUpload.allByteOnDisk) {
+                if (task.transactionId) {
+                    await this.continueUploadOfCompletedVersion(task);
+                }
+                else {
+                    await this.startUploadOfCompletedVersion(task);
+                }
+            }
         }
         else {
-            this.objWriteTaps.delete(task);
-            await this.status.recordTaskCompletion(task);
-        }
-    }
-    async uploadReadyBytes(uploadInfo, tap) {
-        if (tap.cancelledOrErred()) {
-            await this.cancelTransactionIfOpen(uploadInfo);
-            return;
-        }
-        if (uploadInfo.transactionId) {
-            await this.doUploadInTransaction(uploadInfo, tap);
-        }
-        else if (tap.isDone) {
-            // XXX current code does upload after file is written, future code
-            // can start upload sooner, and do diff-ed uploads
-            await this.doFirstUpload(uploadInfo, tap);
+            await this.recordTaskCompletion(task);
         }
     }
     async cancelTransactionIfOpen(uploadInfo) {
@@ -201,62 +281,54 @@ class ObjUpSync {
             }
         });
     }
-    async doFirstUpload(uploadInfo, tap) {
-        assert_1.assert(tap.isDone, `Current implementation works only on completely saved to disk objects`);
-        assert_1.assert(!uploadInfo.awaiting);
+    async startUploadOfCompletedVersion(uploadInfo) {
         const src = await this.obj.getObjSrc(uploadInfo.version);
-        // note that endlessness/finiteness of src is not used
         const header = await src.readHeader();
         const maxChunkLen = (this.remoteStorage.maxChunkSize ?
             this.remoteStorage.maxChunkSize - header.length : MAX_CHUNK_SIZE);
         assert_1.assert(maxChunkLen > 0);
-        const segs = await src.segSrc.read(maxChunkLen);
         const srcLen = (await src.segSrc.getSize()).size;
-        let last;
-        let bytesToSend;
-        if (segs) {
-            if (segs.length === maxChunkLen) {
-                last = (srcLen === segs.length);
-            }
-            else {
-                last = false;
-            }
-            bytesToSend = [header, segs];
+        if (srcLen <= maxChunkLen) {
+            const segs = await src.segSrc.read(maxChunkLen);
+            await this.uploadWholeVersion(!!uploadInfo.createObj, uploadInfo.version, header, segs);
+            uploadInfo.needUpload = undefined;
+            await this.recordTaskCompletion(uploadInfo);
         }
         else {
-            last = true;
-            bytesToSend = header;
+            const segs = await src.segSrc.read(maxChunkLen);
+            uploadInfo.transactionId = await this.startUploadTransaction(!!uploadInfo.createObj, uploadInfo.version, header, segs);
+            uploadInfo.needUpload.segs = [
+                { ofs: segs.length, len: srcLen - segs.length }
+            ];
+            await this.obj.sync().recordInterimStateOfCurrentTask(uploadInfo);
+            this.execPools.add(this);
         }
-        if (last) {
-            const opts = {
-                header: header.length, ver: uploadInfo.version, last
-            };
-            await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytesToSend, opts, undefined);
-            uploadInfo.done = true;
+    }
+    async uploadWholeVersion(create, ver, header, segs) {
+        const opts = {
+            header: header.length, ver, last: true
+        };
+        if (create) {
+            opts.create = true;
         }
-        else {
-            const opts = {
-                header: header.length, ver: uploadInfo.version
-            };
-            const txnId = await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytesToSend, opts, undefined);
-            if (!txnId) {
-                throw new Error(`Server didn't start obj saving transaction`);
-            }
-            uploadInfo.transactionId = txnId;
-            uploadInfo.awaiting = {
-                allByteOnDisk: true,
-                segs: [{ ofs: segs.length, len: srcLen - segs.length }]
-            };
-            this.addToWorker(this);
+        const bytes = (segs ? [header, segs] : [header]);
+        await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytes, opts, undefined);
+    }
+    async startUploadTransaction(create, ver, header, segs) {
+        const opts = {
+            header: header.length, ver
+        };
+        if (create) {
+            opts.create = true;
         }
+        const txnId = await this.remoteStorage.saveNewObjVersion(this.obj.objId, [header, segs], opts, undefined);
+        if (!txnId) {
+            throw new Error(`Server didn't start obj saving transaction`);
+        }
+        return txnId;
     }
-    async doUploadInTransaction(uploadInfo, tap) {
-        assert_1.assert(tap.isDone, `Current implementation works only on completely saved to disk objects`);
-        assert_1.assert(!!uploadInfo.transactionId
-            && !!uploadInfo.awaiting
-            && (uploadInfo.awaiting.segs.length > 0)
-            && (uploadInfo.awaiting.segs[0].len > 0));
-        const section = uploadInfo.awaiting.segs[0];
+    async continueUploadOfCompletedVersion(uploadInfo) {
+        const section = uploadInfo.needUpload.segs[0];
         const lenToRead = Math.min(section.len, (this.remoteStorage.maxChunkSize ?
             this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE));
         const src = await this.obj.getObjSrc(uploadInfo.version);
@@ -265,17 +337,17 @@ class ObjUpSync {
         if (!segs || (segs.length < lenToRead)) {
             throw new Error(`Unexpected end of obj source`);
         }
-        const last = ((uploadInfo.awaiting.segs.length === 0) &&
+        const last = ((uploadInfo.needUpload.segs.length === 1) &&
             (segs.length === section.len));
         if (last) {
             const opts = {
                 ofs: section.ofs,
                 trans: uploadInfo.transactionId,
-                last: true
+                last
             };
             await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
-            uploadInfo.done = true;
-            uploadInfo.awaiting = undefined;
+            uploadInfo.needUpload = undefined;
+            await this.recordTaskCompletion(uploadInfo);
         }
         else {
             const opts = {
@@ -284,41 +356,17 @@ class ObjUpSync {
             };
             await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
             if (segs.length === section.len) {
-                uploadInfo.awaiting.segs.splice(0, 1);
+                uploadInfo.needUpload.segs.splice(0, 1);
             }
             else {
                 section.ofs += segs.length;
                 section.len -= segs.length;
             }
-            this.addToWorker(this);
+            this.execPools.add(this);
         }
     }
 }
 Object.freeze(ObjUpSync.prototype);
 Object.freeze(ObjUpSync);
-class ObjWriteTap {
-    constructor(obj, version, baseVersion, isObjNew) {
-        this.obj = obj;
-        this.version = version;
-        this.baseVersion = baseVersion;
-        this.isObjNew = isObjNew;
-        this.isDone = false;
-        this.isCancelled = false;
-        this.err = undefined;
-        Object.seal(this);
-    }
-    tapOperator() {
-        return operators_1.tap(w => { }, err => {
-            this.err = err;
-            this.isDone = true;
-        }, () => {
-            this.isDone = true;
-        });
-    }
-    cancelledOrErred() {
-        return (this.isCancelled || (this.isDone && this.err));
-    }
-}
-Object.freeze(ObjWriteTap.prototype);
-Object.freeze(ObjWriteTap);
+function noop() { }
 Object.freeze(exports);