core-3nweb-client-lib 0.26.1 → 0.27.3

package/build/core/storage/synced/upsyncer.js

@@ -17,356 +17,286 @@
 */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.UpSyncer = void 0;
-const operators_1 = require("rxjs/operators");
 const labelled_exec_pools_1 = require("../../../lib-common/processes/labelled-exec-pools");
+const exceptions_1 = require("../../../lib-client/3nstorage/exceptions");
 const assert_1 = require("../../../lib-common/assert");
+const deferred_1 = require("../../../lib-common/processes/deferred");
+const buffer_utils_1 = require("../../../lib-common/buffer-utils");
 const MAX_CHUNK_SIZE = 512 * 1024;
 const MAX_FAST_UPLOAD = 2 * 1024 * 1024;
-function executorLabelFor(info) {
-    let uploadSize = 0;
-    if (info.header) {
-        uploadSize += info.header;
-    }
-    for (const seg of info.segs) {
-        uploadSize += seg.len;
-    }
-    return ((uploadSize <= MAX_FAST_UPLOAD) ? 'fast' : 'long');
-}
-function addEventToInfo(info, writeEv) {
-    for (const w of writeEv) {
-        if (w.isHeader) {
-            info.header = w.bytes.length;
-        }
-        else {
-            mergeSections(info.segs, w);
-        }
-    }
-}
-function mergeSections(segs, w) {
-    const wStart = w.ofs;
-    const wLen = w.bytes.length;
-    for (let i = 0; i < segs.length; i += 1) {
-        const section = segs[i];
-        const sectionEnd = section.ofs + section.len;
-        if (sectionEnd < wStart) {
-            continue;
-        }
-        if (sectionEnd === wStart) {
-            section.len += wLen;
-        }
-        else if (section.ofs <= wStart) {
-            const maxOverlap = section.len - (wStart - section.ofs);
-            if (maxOverlap < wLen) {
-                section.len += wLen - maxOverlap;
-            }
-        }
-        else if ((wStart + wLen) === section.ofs) {
-            section.ofs = wStart;
-        }
-        else {
-            segs.splice(i, 0, { ofs: wStart, len: wLen });
-        }
-        return;
-    }
-    segs.push({ ofs: wStart, len: wLen });
-}
 class UpSyncer {
-    constructor(remoteStorage, logError, broadcastUpSyncEvent) {
+    constructor(remoteStorage, logError) {
         this.remoteStorage = remoteStorage;
         this.logError = logError;
-        this.broadcastUpSyncEvent = broadcastUpSyncEvent;
-        this.uploads = new Map();
         this.execPools = new labelled_exec_pools_1.LabelledExecPools([
             { label: 'long', maxProcs: 1 },
             { label: 'fast', maxProcs: 1 }
         ], logError);
         Object.seal(this);
     }
-    getOrMakeUploadsFor(obj) {
-        let uploads = this.uploads.get(obj);
-        if (!uploads) {
-            uploads = new ObjUpSync(obj, this.remoteStorage, this.execPools, this.logError, this.broadcastUpSyncEvent);
-            this.uploads.set(obj, uploads);
-        }
-        return uploads;
-    }
     start() {
         this.execPools.start();
     }
     async stop() {
         await this.execPools.stop(); // implicitly cancels all upsync tasks
-        this.uploads.clear();
     }
+    /**
+     * Creates an rxjs operator to tap saving process, starting upload while
+     * writing is ongoing.
+     */
     tapFileWrite(obj, isNew, newVersion, baseVersion) {
-        const objUploads = this.getOrMakeUploadsFor(obj);
-        return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
+        throw new Error('UpSyncer.tapFileWrite() not implemented');
+        // const objUploads = this.getOrMakeUploadsFor(obj);
+        // return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
     }
     async removeCurrentVersionOf(obj) {
-        const objUploads = this.getOrMakeUploadsFor(obj);
-        if (objUploads.neededExecutor()) {
-            this.execPools.add(objUploads);
+        try {
+            await this.remoteStorage.deleteObj(obj.objId);
         }
+        catch (exc) {
+            // XXX
+            //  - we need to distinguish errors and put this work somewhere
+            //    to run when we go online, for example.
+            await this.logError(exc, `Uploading of obj removal failed.`);
+            return;
+        }
+        await obj.recordRemovalUploadAndGC();
+    }
+    async uploadFromDisk(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote) {
+        const task = await UploadTask.for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote, this.remoteStorage, this.execPools);
+        this.execPools.add(task);
+        await task.completion();
     }
 }
 exports.UpSyncer = UpSyncer;
 Object.freeze(UpSyncer.prototype);
 Object.freeze(UpSyncer);
-class ObjUpSync {
-    constructor(obj, remoteStorage, execPools, logError, broadcastUpSyncEvent) {
-        this.obj = obj;
+class UploadTask {
+    constructor(remoteStorage, objId, objStatus, src, execPools, info, uploadHeader) {
         this.remoteStorage = remoteStorage;
+        this.objId = objId;
+        this.objStatus = objStatus;
+        this.src = src;
         this.execPools = execPools;
-        this.logError = logError;
-        this.broadcastUpSyncEvent = broadcastUpSyncEvent;
-        this.taskInProcess = undefined;
-        this.isCancelled = false;
-        this.runningProc = undefined;
+        this.info = info;
+        this.uploadHeader = uploadHeader;
+        this.uploadCompletion = (0, deferred_1.defer)();
+        this.execLabel = executorLabelFor(this.info.needUpload);
         Object.seal(this);
     }
-    tapFileWrite(isObjNew, newVersion, baseVersion) {
-        // XXX for now upload is started when write is complete, but tapped code
-        //     layout can adopt upload as before completion.
-        const uploadInfo = {
-            type: 'upload',
-            version: newVersion,
-            baseVersion,
-            needUpload: {
-                segs: []
-            }
-        };
-        if (isObjNew) {
-            uploadInfo.createObj = true;
-        }
-        return operators_1.tap(writeEv => {
-            addEventToInfo(uploadInfo.needUpload, writeEv);
-        }, async (err) => {
-            // XXX cancel in-tap upload, when it will be added.
-            //     Something should be done in case there is a transaction that
-            //     needs  cancelling.
-            await this.cancelTransactionIfOpen(uploadInfo);
-            await this.logError(err);
-        }, () => {
-            uploadInfo.needUpload.allByteOnDisk = true;
-            this.enqueueTask(uploadInfo);
-        });
-    }
-    enqueueTask(task) {
-        this.obj.sync().queueTask(task);
-        if (this.neededExecutor()) {
-            this.execPools.add(this);
+    static async for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createObj, remoteStorage, execPools) {
+        const src = await obj.getObjSrcFromLocalAndSyncedBranch(localVersion);
+        let needUpload;
+        if (syncedBase) {
+            const { diff, newSegsPackOrder } = await obj.diffForUploadOf(localVersion);
+            needUpload = await diffVerUpload(src, uploadHeader, diff, newSegsPackOrder);
         }
         else {
-            this.obj.scheduleSelfGC();
-        }
-    }
-    async recordTaskCompletion(task) {
-        this.taskInProcess = undefined;
-        if (task.type === 'upload') {
-            await this.obj.markVersionSynced(task.version);
-        }
-        await this.obj.sync().recordTaskCompletion(task);
-        this.broadcastUpSyncEvent(this.obj.objId, task);
+            needUpload = await wholeVerUpload(src, uploadHeader, createObj);
+        }
+        const info = {
+            type: 'new-version',
+            localVersion,
+            uploadVersion,
+            baseVersion: syncedBase,
+            needUpload
+        };
+        const objStatus = obj.statusObj();
+        await objStatus.recordUploadStart(info);
+        return new UploadTask(remoteStorage, obj.objId, objStatus, src, execPools, info, uploadHeader);
     }
     neededExecutor() {
-        const task = this.obj.sync().glanceOnNextTask();
-        if (!task) {
-            return;
-        }
-        if (task.type !== 'upload') {
-            return 'fast';
-        }
-        if (!task.needUpload) {
-            return;
-        }
-        return executorLabelFor(task.needUpload);
-    }
-    removeArchivedVersion(version) {
-        this.enqueueTask({
-            type: 'removal',
-            archivedVersions: version
-        });
+        return (!this.info.needUpload ? undefined : this.execLabel);
     }
-    async cancel() {
-        var _a;
-        this.isCancelled = true;
-        await ((_a = this.runningProc) === null || _a === void 0 ? void 0 : _a.catch(noop));
+    completion() {
+        return this.uploadCompletion.promise;
     }
     async process() {
-        if (this.obj.sync().isSyncDone() || this.isCancelled) {
+        if (!this.info.needUpload) {
             return;
         }
-        let task = undefined;
         try {
-            if (!this.taskInProcess) {
-                this.taskInProcess = (await this.obj.sync().getTaskForProcessing());
-                if (this.isCancelled) {
-                    return;
+            const upload = this.info.needUpload;
+            if (upload.type === 'ordered-diff') {
+                if (upload.transactionId) {
+                    await this.continueOrderedDiffUpload(upload);
+                }
+                else {
+                    await this.startOrderedDiffUpload(upload);
                 }
             }
-            task = this.taskInProcess;
-        }
-        catch (err) {
-            await this.logError(err, `ObjUpSync.process fails to get task`);
-            return;
-        }
-        try {
-            if (task.type === 'upload') {
-                this.runningProc = this.processUpload(task);
-            }
-            else if (task.type === 'removal') {
-                this.runningProc = this.processRemoval(task);
-            }
-            else if (task.type === 'archiving') {
-                this.runningProc = this.processArchival(task);
+            else if (upload.type === 'ordered-whole') {
+                if (upload.transactionId) {
+                    await this.continueOrderedUpload(upload);
+                }
+                else {
+                    await this.startOrderedUpload(upload);
+                }
             }
             else {
-                throw new Error(`This shouldn't be reached`);
-            }
-            try {
-                await this.runningProc;
+                throw new Error(`Unimplemented ${upload.type} upload`);
             }
-            finally {
-                this.runningProc = undefined;
-            }
-        }
-        catch (err) {
-            await this.logError(err, `From ObjUpSync.process task ${task.type}`);
-        }
-    }
-    async processArchival(task) {
-        // XXX tell server to archive current version
-        await this.logError(`Archival of current version is not implemented, yet.`);
-        await this.recordTaskCompletion(task);
-    }
-    async processRemoval(task) {
-        if (task.currentVersion) {
-            if (this.obj.objId) {
-                await this.remoteStorage.deleteObj(this.obj.objId);
+            await this.objStatus.recordUploadInterimState(this.info);
+            if (this.info.needUpload) {
+                this.execPools.add(this);
             }
             else {
-                await this.logError(`Root obj can't be removed`);
+                this.uploadCompletion.resolve();
             }
         }
-        if (task.archivedVersions) {
-            // XXX tell server to remove given archived versions
-            await this.logError(`Removal of archived version is not implemented, yet.`);
+        catch (exc) {
+            this.info.needUpload = undefined;
+            this.uploadCompletion.reject((0, exceptions_1.makeFSSyncException)(`obj-upload`, {
+                message: `Fail to upload local version ${this.info.uploadVersion}`,
+                localVersion: this.info.uploadVersion,
+                cause: exc
+            }));
+            await this.objStatus.recordUploadCancellation(this.info);
         }
-        await this.recordTaskCompletion(task);
-        this.obj.scheduleSelfGC();
     }
-    async processUpload(task) {
-        if (task.needUpload) {
-            if (task.needUpload.allByteOnDisk) {
-                if (task.transactionId) {
-                    await this.continueUploadOfCompletedVersion(task);
-                }
-                else {
-                    await this.startUploadOfCompletedVersion(task);
-                }
-            }
+    async startOrderedUpload(upload) {
+        const maxSegs = this.maxUploadChunk() - upload.header;
+        (0, assert_1.assert)(maxSegs > 1);
+        const segsToUpload = Math.min(upload.segsLeft, maxSegs);
+        const header = await this.headerToUpload();
+        let segs = undefined;
+        if (segsToUpload > 0) {
+            await this.src.segSrc.seek(upload.segsOfs);
+            segs = await this.src.segSrc.read(segsToUpload);
+        }
+        (0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
+        const ver = this.info.uploadVersion;
+        if (segsToUpload === upload.segsLeft) {
+            await this.remoteStorage.saveNewObjVersion(this.objId, { ver, last: true }, undefined, { header, segs });
+            this.info.needUpload = undefined;
         }
         else {
-            await this.recordTaskCompletion(task);
+            upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver }, undefined, { header, segs });
+            if (!upload.transactionId) {
+                // XXX should this be runtime exception saying that remote acts badly ?
+                throw new Error(`Server didn't start obj saving transaction`);
+            }
+            upload.header = undefined;
+            upload.segsOfs += segsToUpload;
+            upload.segsLeft -= segsToUpload;
         }
     }
-    async cancelTransactionIfOpen(uploadInfo) {
-        const txnId = uploadInfo.transactionId;
-        if (!txnId) {
-            return;
-        }
-        await this.remoteStorage.cancelTransaction(this.obj.objId, txnId)
-            .catch((exc) => {
-            if (!exc.unknownTransaction) {
-                throw exc;
-            }
-        });
+    async headerToUpload() {
+        return (this.uploadHeader ?
+            this.uploadHeader : await this.src.readHeader());
     }
-    async startUploadOfCompletedVersion(uploadInfo) {
-        const src = await this.obj.getObjSrc(uploadInfo.version);
-        const header = await src.readHeader();
-        const maxChunkLen = (this.remoteStorage.maxChunkSize ?
-            this.remoteStorage.maxChunkSize - header.length : MAX_CHUNK_SIZE);
-        assert_1.assert(maxChunkLen > 0);
-        const srcLen = (await src.segSrc.getSize()).size;
-        if (srcLen <= maxChunkLen) {
-            const segs = await src.segSrc.read(maxChunkLen);
-            await this.uploadWholeVersion(!!uploadInfo.createObj, uploadInfo.version, header, segs);
-            uploadInfo.needUpload = undefined;
-            await this.recordTaskCompletion(uploadInfo);
+    async startOrderedDiffUpload(upload) {
+        const diff = buffer_utils_1.utf8.pack(JSON.stringify(upload.diff));
+        const maxSegs = this.maxUploadChunk() - upload.header - diff.length;
+        (0, assert_1.assert)(maxSegs > 1);
+        const header = await this.headerToUpload();
+        const ver = this.info.uploadVersion;
+        if (upload.newSegsLeft.length === 0) {
+            await this.remoteStorage.saveNewObjVersion(this.objId, { ver, last: true }, undefined, { header, diff });
+            this.info.needUpload = undefined;
         }
         else {
-            const segs = await src.segSrc.read(maxChunkLen);
-            uploadInfo.transactionId = await this.startUploadTransaction(!!uploadInfo.createObj, uploadInfo.version, header, segs);
-            uploadInfo.needUpload.segs = [
-                { ofs: segs.length, len: srcLen - segs.length }
-            ];
-            await this.obj.sync().recordInterimStateOfCurrentTask(uploadInfo);
-            this.execPools.add(this);
+            upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver }, undefined, { header, diff });
+            if (!upload.transactionId) {
+                // XXX should this be runtime exception saying that remote acts badly ?
+                throw new Error(`Server didn't start obj saving transaction`);
+            }
+            upload.header = undefined;
         }
     }
-    async uploadWholeVersion(create, ver, header, segs) {
-        const opts = {
-            header: header.length, ver, last: true
-        };
-        if (create) {
-            opts.create = true;
-        }
-        const bytes = (segs ? [header, segs] : [header]);
-        await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytes, opts, undefined);
+    maxUploadChunk() {
+        return (this.remoteStorage.maxChunkSize ?
+            this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE);
     }
-    async startUploadTransaction(create, ver, header, segs) {
-        const opts = {
-            header: header.length, ver
-        };
-        if (create) {
-            opts.create = true;
+    async continueOrderedUpload(upload) {
+        const segsToUpload = Math.min(upload.segsLeft, this.maxUploadChunk());
+        await this.src.segSrc.seek(upload.segsOfs);
+        const segs = await this.src.segSrc.read(segsToUpload);
+        (0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
+        const ofs = upload.segsOfs;
+        const trans = upload.transactionId;
+        if (segsToUpload === upload.segsLeft) {
+            await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
+            this.info.needUpload = undefined;
         }
-        const txnId = await this.remoteStorage.saveNewObjVersion(this.obj.objId, [header, segs], opts, undefined);
-        if (!txnId) {
-            throw new Error(`Server didn't start obj saving transaction`);
+        else {
+            await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
+            upload.segsOfs += segsToUpload;
+            upload.segsLeft -= segsToUpload;
         }
-        return txnId;
     }
-    async continueUploadOfCompletedVersion(uploadInfo) {
-        const section = uploadInfo.needUpload.segs[0];
-        const lenToRead = Math.min(section.len, (this.remoteStorage.maxChunkSize ?
-            this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE));
-        const src = await this.obj.getObjSrc(uploadInfo.version);
-        await src.segSrc.seek(section.ofs);
-        const segs = await src.segSrc.read(lenToRead);
-        if (!segs || (segs.length < lenToRead)) {
-            throw new Error(`Unexpected end of obj source`);
-        }
-        const last = ((uploadInfo.needUpload.segs.length === 1) &&
-            (segs.length === section.len));
-        if (last) {
-            const opts = {
-                ofs: section.ofs,
-                trans: uploadInfo.transactionId,
-                last
-            };
-            await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
-            uploadInfo.needUpload = undefined;
-            await this.recordTaskCompletion(uploadInfo);
+    async continueOrderedDiffUpload(upload) {
+        const maxSegs = this.maxUploadChunk();
+        const segInfo = upload.newSegsLeft[0];
+        (0, assert_1.assert)(!!segInfo);
+        const len = Math.min(maxSegs, segInfo.len);
+        const segs = await this.src.segSrc.read(len);
+        (0, assert_1.assert)(!!segs && (segs.length === len));
+        if (segInfo.len > len) {
+            upload.newSegsLeft.splice(1, 0, {
+                len: segInfo.len - len,
+                thisVerOfs: segInfo.thisVerOfs + len
+            });
+        }
+        const ofs = segInfo.thisVerOfs;
+        const trans = upload.transactionId;
+        if (upload.newSegsLeft.length === 1) {
+            await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
+            this.info.needUpload = undefined;
         }
         else {
-            const opts = {
-                ofs: section.ofs,
-                trans: uploadInfo.transactionId
-            };
-            await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
-            if (segs.length === section.len) {
-                uploadInfo.needUpload.segs.splice(0, 1);
-            }
-            else {
-                section.ofs += segs.length;
-                section.len -= segs.length;
-            }
-            this.execPools.add(this);
+            await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
+            upload.newSegsLeft.splice(0, 1);
         }
     }
+    cancel() {
+        // XXX
+        throw new Error("UploadTask.cancel() not implemented.");
+    }
+}
+Object.freeze(UploadTask.prototype);
+Object.freeze(UploadTask);
+async function wholeVerUpload(src, uploadHeader, createObj) {
+    const header = await expectedHeaderLen(src, uploadHeader);
+    const segsSize = (await src.segSrc.getSize()).size;
+    return {
+        type: 'ordered-whole',
+        createObj,
+        header,
+        segsOfs: 0,
+        segsLeft: segsSize
+    };
+}
+async function diffVerUpload(src, uploadHeader, diff, newSegsPackOrder) {
+    const header = await expectedHeaderLen(src, uploadHeader);
+    return {
+        type: 'ordered-diff',
+        diff,
+        newSegsLeft: newSegsPackOrder,
+        header
+    };
+}
+async function expectedHeaderLen(src, uploadHeader) {
+    return (uploadHeader ?
+        uploadHeader.length : (await src.readHeader()).length);
+}
+function uploadSize(info) {
+    if (info.type === 'ordered-whole') {
+        return info.header + info.segsLeft;
+    }
+    else if (info.type === 'ordered-diff') {
+        let uploadSize = info.header;
+        for (const { len } of info.newSegsLeft) {
+            uploadSize += len;
+        }
+        return uploadSize;
+    }
+    else {
+        throw new Error(`Unimplemented upload type ${info.type}`);
+    }
+}
+function executorLabelFor(info) {
+    return ((uploadSize(info) <= MAX_FAST_UPLOAD) ? 'fast' : 'long');
 }
-Object.freeze(ObjUpSync.prototype);
-Object.freeze(ObjUpSync);
 function noop() { }
 Object.freeze(exports);

package/build/core/storage/system-folders/apps-data.d.ts

@@ -0,0 +1,16 @@
+declare type WritableFS = web3n.files.WritableFS;
+export declare class AppDataFolders {
+    private readonly fs;
+    private writingSync;
+    private syncFolderProc;
+    private constructor();
+    static make(rootFS: WritableFS): Promise<AppDataFolders>;
+    getOrMake(folder: string): Promise<WritableFS>;
+    private syncP;
+    private makeSyncedFolder;
+    private syncBeforeChange;
+    private uploadAfterCreationOf;
+    private startSyncProc;
+    stopSync(): void;
+}
+export {};