core-3nweb-client-lib 0.25.6 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/api-defs/asmail.d.ts +1 -1
- package/build/api-defs/files.d.ts +282 -70
- package/build/core/app-files.js +7 -7
- package/build/core/asmail/config/common.js +4 -4
- package/build/core/asmail/config/index.js +2 -2
- package/build/core/asmail/config/published-intro-key.js +1 -1
- package/build/core/asmail/delivery/common.js +7 -7
- package/build/core/asmail/delivery/index.js +7 -6
- package/build/core/asmail/delivery/msg.js +8 -7
- package/build/core/asmail/delivery/per-recipient-wip.js +3 -3
- package/build/core/asmail/inbox/attachments/fs.js +11 -1
- package/build/core/asmail/inbox/cached-msgs.js +3 -3
- package/build/core/asmail/inbox/inbox-events.js +5 -4
- package/build/core/asmail/inbox/index.js +12 -12
- package/build/core/asmail/inbox/msg-downloader.js +3 -3
- package/build/core/asmail/inbox/msg-indexing.js +4 -4
- package/build/core/asmail/inbox/msg-on-disk.js +7 -7
- package/build/core/asmail/index.d.ts +3 -3
- package/build/core/asmail/index.js +13 -8
- package/build/core/asmail/key-verification.js +5 -5
- package/build/core/asmail/keyring/common.js +7 -6
- package/build/core/asmail/keyring/correspondent-keys.js +8 -7
- package/build/core/asmail/keyring/id-to-email-map.js +2 -1
- package/build/core/asmail/keyring/index.d.ts +7 -8
- package/build/core/asmail/keyring/index.js +15 -14
- package/build/core/asmail/keyring/keyring-storage.js +4 -3
- package/build/core/asmail/msg/opener.js +3 -3
- package/build/core/asmail/msg/packer.js +13 -13
- package/build/core/asmail/sending-params/own-params.js +4 -4
- package/build/core/asmail/sending-params/params-from-others.js +3 -3
- package/build/core/id-manager.js +8 -5
- package/build/core/index.d.ts +2 -1
- package/build/core/index.js +14 -14
- package/build/core/sign-in.d.ts +5 -4
- package/build/core/sign-in.js +12 -14
- package/build/core/sign-up.d.ts +1 -0
- package/build/core/sign-up.js +15 -11
- package/build/core/storage/common/json-saving.d.ts +21 -0
- package/build/core/storage/common/json-saving.js +82 -0
- package/build/core/storage/common/obj-info-file.d.ts +51 -0
- package/build/core/storage/common/obj-info-file.js +153 -5
- package/build/core/storage/common/utils.d.ts +2 -0
- package/build/core/storage/common/utils.js +32 -0
- package/build/core/storage/index.d.ts +3 -17
- package/build/core/storage/index.js +57 -77
- package/build/core/storage/local/obj-files-gc.d.ts +2 -0
- package/build/core/storage/local/obj-files-gc.js +53 -39
- package/build/core/storage/local/obj-files.d.ts +6 -9
- package/build/core/storage/local/obj-files.js +16 -19
- package/build/core/storage/local/obj-status.d.ts +20 -30
- package/build/core/storage/local/obj-status.js +46 -113
- package/build/core/storage/local/storage.d.ts +15 -5
- package/build/core/storage/local/storage.js +37 -18
- package/build/core/storage/synced/downloader.js +7 -6
- package/build/core/storage/synced/obj-files-gc.d.ts +6 -1
- package/build/core/storage/synced/obj-files-gc.js +106 -13
- package/build/core/storage/synced/obj-files.d.ts +46 -47
- package/build/core/storage/synced/obj-files.js +207 -154
- package/build/core/storage/synced/obj-status.d.ts +103 -42
- package/build/core/storage/synced/obj-status.js +525 -137
- package/build/core/storage/synced/remote-events.d.ts +11 -12
- package/build/core/storage/synced/remote-events.js +80 -57
- package/build/core/storage/synced/storage.d.ts +24 -5
- package/build/core/storage/synced/storage.js +123 -38
- package/build/core/storage/synced/upload-header-file.d.ts +4 -0
- package/build/core/storage/synced/upload-header-file.js +64 -0
- package/build/core/storage/synced/upsyncer.d.ts +15 -9
- package/build/core/storage/synced/upsyncer.js +219 -246
- package/build/core/storage/system-folders/apps-data.d.ts +16 -0
- package/build/core/storage/system-folders/apps-data.js +110 -0
- package/build/core/storage/system-folders/index.d.ts +18 -0
- package/build/core/storage/system-folders/index.js +77 -0
- package/build/core-ipc/common-caps.js +3 -3
- package/build/core-ipc/generic.js +8 -8
- package/build/core-ipc/startup-caps.js +2 -2
- package/build/cryptors.js +6 -2
- package/build/ipc-via-protobuf/asmail-cap.js +67 -83
- package/build/ipc-via-protobuf/bytes.js +16 -17
- package/build/ipc-via-protobuf/connector-clients-side.d.ts +3 -0
- package/build/ipc-via-protobuf/connector-clients-side.js +62 -25
- package/build/ipc-via-protobuf/connector-services-side.js +10 -10
- package/build/ipc-via-protobuf/connector.js +4 -4
- package/build/ipc-via-protobuf/file.d.ts +48 -12
- package/build/ipc-via-protobuf/file.js +476 -120
- package/build/ipc-via-protobuf/fs.d.ts +8 -0
- package/build/ipc-via-protobuf/fs.js +592 -159
- package/build/ipc-via-protobuf/log-cap.js +2 -2
- package/build/ipc-via-protobuf/mailerid.js +3 -3
- package/build/ipc-via-protobuf/protobuf-msg.d.ts +1 -0
- package/build/ipc-via-protobuf/protobuf-msg.js +11 -7
- package/build/ipc-via-protobuf/startup-cap.js +23 -23
- package/build/ipc-via-protobuf/storage-cap.js +12 -12
- package/build/ipc.js +7 -2
- package/build/lib-client/3nstorage/exceptions.d.ts +12 -8
- package/build/lib-client/3nstorage/exceptions.js +31 -10
- package/build/lib-client/3nstorage/service.d.ts +16 -2
- package/build/lib-client/3nstorage/service.js +109 -39
- package/build/lib-client/3nstorage/util/file-based-json.d.ts +2 -1
- package/build/lib-client/3nstorage/util/file-based-json.js +1 -1
- package/build/lib-client/3nstorage/xsp-fs/attrs.js +17 -17
- package/build/lib-client/3nstorage/xsp-fs/common.d.ts +52 -14
- package/build/lib-client/3nstorage/xsp-fs/common.js +31 -16
- package/build/lib-client/3nstorage/xsp-fs/file-node.d.ts +1 -0
- package/build/lib-client/3nstorage/xsp-fs/file-node.js +18 -14
- package/build/lib-client/3nstorage/xsp-fs/file.d.ts +31 -6
- package/build/lib-client/3nstorage/xsp-fs/file.js +74 -23
- package/build/lib-client/3nstorage/xsp-fs/folder-node-serialization.js +4 -4
- package/build/lib-client/3nstorage/xsp-fs/folder-node.d.ts +24 -11
- package/build/lib-client/3nstorage/xsp-fs/folder-node.js +599 -189
- package/build/lib-client/3nstorage/xsp-fs/fs.d.ts +45 -9
- package/build/lib-client/3nstorage/xsp-fs/fs.js +326 -74
- package/build/lib-client/3nstorage/xsp-fs/link-node.d.ts +1 -0
- package/build/lib-client/3nstorage/xsp-fs/link-node.js +7 -2
- package/build/lib-client/3nstorage/xsp-fs/node-in-fs.d.ts +30 -20
- package/build/lib-client/3nstorage/xsp-fs/node-in-fs.js +239 -106
- package/build/lib-client/3nstorage/xsp-fs/node-persistence.d.ts +1 -1
- package/build/lib-client/3nstorage/xsp-fs/node-persistence.js +18 -19
- package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v1.js +5 -5
- package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v2.js +56 -56
- package/build/lib-client/3nweb-signup.js +4 -4
- package/build/lib-client/asmail/recipient.js +15 -15
- package/build/lib-client/asmail/sender.js +22 -22
- package/build/lib-client/asmail/service-config.js +3 -3
- package/build/lib-client/cryptor/cryptor-in-worker.js +19 -17
- package/build/lib-client/cryptor/cryptor-wasm.js +1 -1
- package/build/lib-client/cryptor/cryptor.js +4 -2
- package/build/lib-client/cryptor/cryptor.wasm +0 -0
- package/build/lib-client/cryptor/in-proc-js.js +1 -1
- package/build/lib-client/cryptor/in-proc-wasm.js +7 -7
- package/build/lib-client/cryptor/worker-js.js +2 -2
- package/build/lib-client/cryptor/worker-wasm.js +2 -2
- package/build/lib-client/files-select.js +1 -1
- package/build/lib-client/files.d.ts +1 -1
- package/build/lib-client/files.js +71 -4
- package/build/lib-client/fs-collection.js +3 -2
- package/build/lib-client/fs-sync-utils.d.ts +5 -0
- package/build/lib-client/fs-sync-utils.js +61 -0
- package/build/lib-client/fs-view.d.ts +14 -0
- package/build/lib-client/fs-view.js +33 -0
- package/build/lib-client/key-derivation.js +1 -1
- package/build/lib-client/local-files/dev-file-sink.js +9 -9
- package/build/lib-client/local-files/dev-file-src.js +2 -2
- package/build/lib-client/local-files/device-fs.d.ts +1 -1
- package/build/lib-client/local-files/device-fs.js +56 -54
- package/build/lib-client/logging/log-to-file.d.ts +1 -1
- package/build/lib-client/logging/log-to-file.js +11 -11
- package/build/lib-client/mailer-id/login.js +7 -7
- package/build/lib-client/mailer-id/provisioner.js +12 -12
- package/build/lib-client/objs-on-disk/file-writing-proc.js +5 -5
- package/build/lib-client/objs-on-disk/obj-folders.js +33 -33
- package/build/lib-client/objs-on-disk/obj-on-disk.d.ts +13 -2
- package/build/lib-client/objs-on-disk/obj-on-disk.js +24 -9
- package/build/lib-client/request-utils.d.ts +1 -0
- package/build/lib-client/request-utils.js +14 -14
- package/build/lib-client/server-events.d.ts +3 -3
- package/build/lib-client/server-events.js +12 -10
- package/build/lib-client/service-locator.js +10 -10
- package/build/lib-client/user-with-mid-session.js +7 -7
- package/build/lib-client/user-with-pkl-session.js +25 -25
- package/build/lib-client/ws-utils.js +3 -3
- package/build/lib-common/async-cryptor-wrap.js +4 -4
- package/build/lib-common/async-fs-node.d.ts +5 -3
- package/build/lib-common/async-fs-node.js +19 -18
- package/build/lib-common/byte-streaming/pipe.js +1 -1
- package/build/lib-common/byte-streaming/wrapping.js +17 -17
- package/build/lib-common/canonical-address.js +1 -1
- package/build/lib-common/exceptions/error.d.ts +1 -0
- package/build/lib-common/exceptions/error.js +7 -6
- package/build/lib-common/exceptions/file.js +10 -1
- package/build/lib-common/ipc/generic-ipc.js +2 -2
- package/build/lib-common/ipc/ws-ipc.js +2 -2
- package/build/lib-common/json-utils.js +2 -1
- package/build/lib-common/mid-sigs-NaCl-Ed.js +14 -14
- package/build/lib-common/objs-on-disk/file-layout.d.ts +19 -0
- package/build/lib-common/objs-on-disk/file-layout.js +130 -12
- package/build/lib-common/objs-on-disk/obj-file.d.ts +13 -2
- package/build/lib-common/objs-on-disk/obj-file.js +99 -37
- package/build/lib-common/objs-on-disk/utils.d.ts +1 -0
- package/build/lib-common/objs-on-disk/utils.js +4 -4
- package/build/lib-common/objs-on-disk/v1-obj-file-format.js +14 -14
- package/build/lib-common/processes/deferred.d.ts +6 -0
- package/build/lib-common/processes/deferred.js +30 -0
- package/build/lib-common/processes/labelled-exec-pools.d.ts +33 -0
- package/build/lib-common/processes/labelled-exec-pools.js +141 -0
- package/build/lib-common/processes/pressure.d.ts +7 -0
- package/build/lib-common/processes/pressure.js +56 -0
- package/build/lib-common/processes/sleep.d.ts +1 -0
- package/build/lib-common/processes/sleep.js +26 -0
- package/build/lib-common/{processes.d.ts → processes/synced.d.ts} +0 -40
- package/build/lib-common/{processes.js → processes/synced.js} +187 -204
- package/build/lib-common/processes/timeout.d.ts +1 -0
- package/build/lib-common/processes/timeout.js +51 -0
- package/build/lib-common/random-node.js +7 -7
- package/build/lib-common/service-api/3nstorage/owner.d.ts +100 -39
- package/build/lib-common/service-api/3nstorage/owner.js +85 -42
- package/build/lib-common/service-api/asmail/delivery.js +2 -2
- package/build/lib-common/service-api/asmail/retrieval.js +1 -1
- package/build/lib-common/timed-cache.d.ts +1 -0
- package/build/lib-common/timed-non-weak-cache.d.ts +1 -0
- package/build/lib-common/timed-non-weak-cache.js +11 -0
- package/build/lib-common/utils-for-observables.d.ts +15 -1
- package/build/lib-common/utils-for-observables.js +70 -19
- package/build/lib-common/weak-cache.d.ts +1 -0
- package/build/lib-common/weak-cache.js +12 -1
- package/build/lib-index.d.ts +2 -1
- package/build/lib-index.js +10 -7
- package/build/protos/asmail.proto.js +12912 -7127
- package/build/protos/file.proto.js +4848 -2399
- package/build/protos/fs.proto.js +9230 -3445
- package/package.json +8 -7
- package/protos/file.proto +91 -11
- package/protos/fs.proto +107 -8
- package/build/core/storage/synced/upsync-status.d.ts +0 -41
- package/build/core/storage/synced/upsync-status.js +0 -158
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/*
|
|
3
|
-
Copyright (C) 2020 3NSoft Inc.
|
|
3
|
+
Copyright (C) 2020, 2022 3NSoft Inc.
|
|
4
4
|
|
|
5
5
|
This program is free software: you can redistribute it and/or modify it under
|
|
6
6
|
the terms of the GNU General Public License as published by the Free Software
|
|
@@ -13,312 +13,285 @@
|
|
|
13
13
|
See the GNU General Public License for more details.
|
|
14
14
|
|
|
15
15
|
You should have received a copy of the GNU General Public License along with
|
|
16
|
-
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
17
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
19
|
exports.UpSyncer = void 0;
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const upsync_status_1 = require("./upsync-status");
|
|
20
|
+
const labelled_exec_pools_1 = require("../../../lib-common/processes/labelled-exec-pools");
|
|
21
|
+
const exceptions_1 = require("../../../lib-client/3nstorage/exceptions");
|
|
22
22
|
const assert_1 = require("../../../lib-common/assert");
|
|
23
|
+
const deferred_1 = require("../../../lib-common/processes/deferred");
|
|
24
|
+
const buffer_utils_1 = require("../../../lib-common/buffer-utils");
|
|
23
25
|
const MAX_CHUNK_SIZE = 512 * 1024;
|
|
26
|
+
const MAX_FAST_UPLOAD = 2 * 1024 * 1024;
|
|
24
27
|
class UpSyncer {
|
|
25
|
-
constructor(remoteStorage,
|
|
28
|
+
constructor(remoteStorage, logError) {
|
|
26
29
|
this.remoteStorage = remoteStorage;
|
|
27
|
-
this.files = files;
|
|
28
30
|
this.logError = logError;
|
|
29
|
-
this.
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
this.worker = new processes_1.Worker(async (u) => {
|
|
34
|
-
await u.process();
|
|
35
|
-
if (u.isDone() && (u === this.uploads.get(u.obj))) {
|
|
36
|
-
this.uploads.delete(u.obj);
|
|
37
|
-
}
|
|
38
|
-
}, async (queue) => {
|
|
39
|
-
for (const u of queue) {
|
|
40
|
-
u.stop();
|
|
41
|
-
this.uploads.delete(u.obj);
|
|
42
|
-
}
|
|
43
|
-
});
|
|
31
|
+
this.execPools = new labelled_exec_pools_1.LabelledExecPools([
|
|
32
|
+
{ label: 'long', maxProcs: 1 },
|
|
33
|
+
{ label: 'fast', maxProcs: 1 }
|
|
34
|
+
], logError);
|
|
44
35
|
Object.seal(this);
|
|
45
36
|
}
|
|
46
|
-
getOrMakeUploadsFor(obj) {
|
|
47
|
-
let uploads = this.uploads.get(obj);
|
|
48
|
-
if (!uploads) {
|
|
49
|
-
const status = new upsync_status_1.UpSyncTasks(obj.objFolder, this.logError);
|
|
50
|
-
uploads = new ObjUpSync(obj, status, this.remoteStorage, this.addToWorker, this.logError);
|
|
51
|
-
this.uploads.set(obj, uploads);
|
|
52
|
-
}
|
|
53
|
-
return uploads;
|
|
54
|
-
}
|
|
55
37
|
start() {
|
|
56
|
-
this.
|
|
38
|
+
this.execPools.start();
|
|
57
39
|
}
|
|
58
40
|
async stop() {
|
|
59
|
-
await this.
|
|
60
|
-
for (const upload of this.uploads.values()) {
|
|
61
|
-
upload.stop();
|
|
62
|
-
}
|
|
63
|
-
this.uploads.clear();
|
|
64
|
-
}
|
|
65
|
-
get chunkSize() {
|
|
66
|
-
return (this.remoteStorage.maxChunkSize ?
|
|
67
|
-
Math.min(this.remoteStorage.maxChunkSize, MAX_CHUNK_SIZE) :
|
|
68
|
-
MAX_CHUNK_SIZE);
|
|
41
|
+
await this.execPools.stop(); // implicitly cancels all upsync tasks
|
|
69
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Creates an rxjs operator to tap saving process, starting upload while
|
|
45
|
+
* writing is ongoing.
|
|
46
|
+
*/
|
|
70
47
|
tapFileWrite(obj, isNew, newVersion, baseVersion) {
|
|
71
|
-
|
|
72
|
-
|
|
48
|
+
throw new Error('UpSyncer.tapFileWrite() not implemented');
|
|
49
|
+
// const objUploads = this.getOrMakeUploadsFor(obj);
|
|
50
|
+
// return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
|
|
73
51
|
}
|
|
74
52
|
async removeCurrentVersionOf(obj) {
|
|
75
|
-
|
|
76
|
-
objUploads.
|
|
53
|
+
throw new Error('UpSyncer.removeCurrentVersionOf() not implemented');
|
|
54
|
+
// const objUploads = this.getOrMakeUploadsFor(obj);
|
|
55
|
+
// XXX is removal set in this older code? This may need to change anyway
|
|
56
|
+
// if (objUploads.neededExecutor()) {
|
|
57
|
+
// this.execPools.add(objUploads);
|
|
58
|
+
// }
|
|
59
|
+
}
|
|
60
|
+
async uploadFromDisk(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote) {
|
|
61
|
+
const task = await UploadTask.for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote, this.remoteStorage, this.execPools);
|
|
62
|
+
this.execPools.add(task);
|
|
63
|
+
await task.completion();
|
|
77
64
|
}
|
|
78
65
|
}
|
|
79
66
|
exports.UpSyncer = UpSyncer;
|
|
80
67
|
Object.freeze(UpSyncer.prototype);
|
|
81
68
|
Object.freeze(UpSyncer);
|
|
82
|
-
class
|
|
83
|
-
constructor(
|
|
84
|
-
this.obj = obj;
|
|
85
|
-
this.status = status;
|
|
69
|
+
class UploadTask {
|
|
70
|
+
constructor(remoteStorage, objId, objStatus, src, execPools, info, uploadHeader) {
|
|
86
71
|
this.remoteStorage = remoteStorage;
|
|
87
|
-
this.
|
|
88
|
-
this.
|
|
89
|
-
this.
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
return tap.tapOperator();
|
|
97
|
-
}
|
|
98
|
-
stop() {
|
|
99
|
-
// XXX
|
|
100
|
-
// - is something needed here?
|
|
72
|
+
this.objId = objId;
|
|
73
|
+
this.objStatus = objStatus;
|
|
74
|
+
this.src = src;
|
|
75
|
+
this.execPools = execPools;
|
|
76
|
+
this.info = info;
|
|
77
|
+
this.uploadHeader = uploadHeader;
|
|
78
|
+
this.uploadCompletion = (0, deferred_1.defer)();
|
|
79
|
+
this.execLabel = executorLabelFor(this.info.needUpload);
|
|
80
|
+
Object.seal(this);
|
|
101
81
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
82
|
+
static async for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createObj, remoteStorage, execPools) {
|
|
83
|
+
const src = await obj.getObjSrcFromLocalAndSyncedBranch(localVersion);
|
|
84
|
+
let needUpload;
|
|
85
|
+
if (syncedBase) {
|
|
86
|
+
const { diff, newSegsPackOrder } = await obj.diffForUploadOf(localVersion);
|
|
87
|
+
needUpload = await diffVerUpload(src, uploadHeader, diff, newSegsPackOrder);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
needUpload = await wholeVerUpload(src, uploadHeader, createObj);
|
|
91
|
+
}
|
|
92
|
+
const info = {
|
|
93
|
+
localVersion,
|
|
94
|
+
uploadVersion,
|
|
95
|
+
baseVersion: syncedBase,
|
|
96
|
+
needUpload
|
|
97
|
+
};
|
|
98
|
+
const objStatus = obj.statusObj();
|
|
99
|
+
await objStatus.recordUploadStart(info);
|
|
100
|
+
return new UploadTask(remoteStorage, obj.objId, objStatus, src, execPools, info, uploadHeader);
|
|
108
101
|
}
|
|
109
|
-
|
|
110
|
-
this.
|
|
111
|
-
type: 'removal',
|
|
112
|
-
archivedVersions: version
|
|
113
|
-
});
|
|
114
|
-
this.addToWorker(this);
|
|
102
|
+
neededExecutor() {
|
|
103
|
+
return (!this.info.needUpload ? undefined : this.execLabel);
|
|
115
104
|
}
|
|
116
|
-
|
|
117
|
-
return this.
|
|
105
|
+
completion() {
|
|
106
|
+
return this.uploadCompletion.promise;
|
|
118
107
|
}
|
|
119
108
|
async process() {
|
|
120
|
-
if (this.
|
|
109
|
+
if (!this.info.needUpload) {
|
|
121
110
|
return;
|
|
122
111
|
}
|
|
123
112
|
try {
|
|
124
|
-
const
|
|
125
|
-
if (
|
|
126
|
-
|
|
113
|
+
const upload = this.info.needUpload;
|
|
114
|
+
if (upload.type === 'ordered-diff') {
|
|
115
|
+
if (upload.transactionId) {
|
|
116
|
+
await this.continueOrderedDiffUpload(upload);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
await this.startOrderedDiffUpload(upload);
|
|
120
|
+
}
|
|
127
121
|
}
|
|
128
|
-
else if (
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
122
|
+
else if (upload.type === 'ordered-whole') {
|
|
123
|
+
if (upload.transactionId) {
|
|
124
|
+
await this.continueOrderedUpload(upload);
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
await this.startOrderedUpload(upload);
|
|
128
|
+
}
|
|
133
129
|
}
|
|
134
130
|
else {
|
|
135
|
-
throw new Error(`
|
|
131
|
+
throw new Error(`Unimplemented ${upload.type} upload`);
|
|
136
132
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
}
|
|
141
|
-
if (!this.isDone()) {
|
|
142
|
-
this.addToWorker(this);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
async processArchival(task) {
|
|
146
|
-
// XXX tell server to archive current version
|
|
147
|
-
await this.logError(`Archival of current version is not implemented, yet.`);
|
|
148
|
-
await this.status.recordTaskCompletion(task);
|
|
149
|
-
}
|
|
150
|
-
async processRemoval(task) {
|
|
151
|
-
if (task.currentVersion) {
|
|
152
|
-
if (this.obj.objId) {
|
|
153
|
-
await this.remoteStorage.deleteObj(this.obj.objId);
|
|
133
|
+
await this.objStatus.recordUploadInterimState(this.info);
|
|
134
|
+
if (this.info.needUpload) {
|
|
135
|
+
this.execPools.add(this);
|
|
154
136
|
}
|
|
155
137
|
else {
|
|
156
|
-
|
|
138
|
+
this.uploadCompletion.resolve();
|
|
157
139
|
}
|
|
158
140
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
141
|
+
catch (exc) {
|
|
142
|
+
this.info.needUpload = undefined;
|
|
143
|
+
this.uploadCompletion.reject((0, exceptions_1.makeFSSyncException)(`obj-upload`, {
|
|
144
|
+
message: `Fail to upload local version ${this.info.uploadVersion}`,
|
|
145
|
+
localVersion: this.info.uploadVersion,
|
|
146
|
+
cause: exc
|
|
147
|
+
}));
|
|
148
|
+
await this.objStatus.recordUploadCancellation(this.info);
|
|
162
149
|
}
|
|
163
|
-
await this.status.recordTaskCompletion(task);
|
|
164
150
|
}
|
|
165
|
-
async
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
151
|
+
async startOrderedUpload(upload) {
|
|
152
|
+
const maxSegs = this.maxUploadChunk() - upload.header;
|
|
153
|
+
(0, assert_1.assert)(maxSegs > 1);
|
|
154
|
+
const segsToUpload = Math.min(upload.segsLeft, maxSegs);
|
|
155
|
+
const header = await this.headerToUpload();
|
|
156
|
+
let segs = undefined;
|
|
157
|
+
if (segsToUpload > 0) {
|
|
158
|
+
await this.src.segSrc.seek(upload.segsOfs);
|
|
159
|
+
segs = await this.src.segSrc.read(segsToUpload);
|
|
169
160
|
}
|
|
170
|
-
|
|
171
|
-
|
|
161
|
+
(0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
|
|
162
|
+
const ver = this.info.uploadVersion;
|
|
163
|
+
const create = (upload.createObj ? true : undefined);
|
|
164
|
+
if (segsToUpload === upload.segsLeft) {
|
|
165
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, { ver, create, last: true }, undefined, { header, segs });
|
|
166
|
+
this.info.needUpload = undefined;
|
|
172
167
|
}
|
|
173
168
|
else {
|
|
174
|
-
this.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
if (uploadInfo.transactionId) {
|
|
184
|
-
await this.doUploadInTransaction(uploadInfo, tap);
|
|
185
|
-
}
|
|
186
|
-
else if (tap.isDone) {
|
|
187
|
-
// XXX current code does upload after file is written, future code
|
|
188
|
-
// can start upload sooner, and do diff-ed uploads
|
|
189
|
-
await this.doFirstUpload(uploadInfo, tap);
|
|
169
|
+
upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver, create }, undefined, { header, segs });
|
|
170
|
+
if (!upload.transactionId) {
|
|
171
|
+
// XXX should this be runtime exception saying that remote acts badly ?
|
|
172
|
+
throw new Error(`Server didn't start obj saving transaction`);
|
|
173
|
+
}
|
|
174
|
+
upload.header = undefined;
|
|
175
|
+
upload.segsOfs += segsToUpload;
|
|
176
|
+
upload.segsLeft -= segsToUpload;
|
|
190
177
|
}
|
|
191
178
|
}
|
|
192
|
-
async
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
return;
|
|
196
|
-
}
|
|
197
|
-
await this.remoteStorage.cancelTransaction(this.obj.objId, txnId)
|
|
198
|
-
.catch((exc) => {
|
|
199
|
-
if (!exc.unknownTransaction) {
|
|
200
|
-
throw exc;
|
|
201
|
-
}
|
|
202
|
-
});
|
|
179
|
+
async headerToUpload() {
|
|
180
|
+
return (this.uploadHeader ?
|
|
181
|
+
this.uploadHeader : await this.src.readHeader());
|
|
203
182
|
}
|
|
204
|
-
async
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
this.remoteStorage.
|
|
212
|
-
|
|
213
|
-
const segs = await src.segSrc.read(maxChunkLen);
|
|
214
|
-
const srcLen = (await src.segSrc.getSize()).size;
|
|
215
|
-
let last;
|
|
216
|
-
let bytesToSend;
|
|
217
|
-
if (segs) {
|
|
218
|
-
if (segs.length === maxChunkLen) {
|
|
219
|
-
last = (srcLen === segs.length);
|
|
220
|
-
}
|
|
221
|
-
else {
|
|
222
|
-
last = false;
|
|
223
|
-
}
|
|
224
|
-
bytesToSend = [header, segs];
|
|
183
|
+
async startOrderedDiffUpload(upload) {
|
|
184
|
+
const diff = buffer_utils_1.utf8.pack(JSON.stringify(upload.diff));
|
|
185
|
+
const maxSegs = this.maxUploadChunk() - upload.header - diff.length;
|
|
186
|
+
(0, assert_1.assert)(maxSegs > 1);
|
|
187
|
+
const header = await this.headerToUpload();
|
|
188
|
+
const ver = this.info.uploadVersion;
|
|
189
|
+
if (upload.newSegsLeft.length === 0) {
|
|
190
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, { ver, last: true }, undefined, { header, diff });
|
|
191
|
+
this.info.needUpload = undefined;
|
|
225
192
|
}
|
|
226
193
|
else {
|
|
227
|
-
|
|
228
|
-
|
|
194
|
+
upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver }, undefined, { header, diff });
|
|
195
|
+
if (!upload.transactionId) {
|
|
196
|
+
// XXX should this be runtime exception saying that remote acts badly ?
|
|
197
|
+
throw new Error(`Server didn't start obj saving transaction`);
|
|
198
|
+
}
|
|
199
|
+
upload.header = undefined;
|
|
229
200
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
201
|
+
}
|
|
202
|
+
maxUploadChunk() {
|
|
203
|
+
return (this.remoteStorage.maxChunkSize ?
|
|
204
|
+
this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE);
|
|
205
|
+
}
|
|
206
|
+
async continueOrderedUpload(upload) {
|
|
207
|
+
const segsToUpload = Math.min(upload.segsLeft, this.maxUploadChunk());
|
|
208
|
+
await this.src.segSrc.seek(upload.segsOfs);
|
|
209
|
+
const segs = await this.src.segSrc.read(segsToUpload);
|
|
210
|
+
(0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
|
|
211
|
+
const ofs = upload.segsOfs;
|
|
212
|
+
const trans = upload.transactionId;
|
|
213
|
+
if (segsToUpload === upload.segsLeft) {
|
|
214
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
|
|
215
|
+
this.info.needUpload = undefined;
|
|
236
216
|
}
|
|
237
217
|
else {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
const txnId = await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytesToSend, opts, undefined);
|
|
242
|
-
if (!txnId) {
|
|
243
|
-
throw new Error(`Server didn't start obj saving transaction`);
|
|
244
|
-
}
|
|
245
|
-
uploadInfo.transactionId = txnId;
|
|
246
|
-
uploadInfo.awaiting = {
|
|
247
|
-
allByteOnDisk: true,
|
|
248
|
-
segs: [{ ofs: segs.length, len: srcLen - segs.length }]
|
|
249
|
-
};
|
|
250
|
-
this.addToWorker(this);
|
|
218
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
|
|
219
|
+
upload.segsOfs += segsToUpload;
|
|
220
|
+
upload.segsLeft -= segsToUpload;
|
|
251
221
|
}
|
|
252
222
|
}
|
|
253
|
-
async
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
if (!segs || (segs.length < lenToRead)) {
|
|
266
|
-
throw new Error(`Unexpected end of obj source`);
|
|
223
|
+
async continueOrderedDiffUpload(upload) {
|
|
224
|
+
const maxSegs = this.maxUploadChunk();
|
|
225
|
+
const segInfo = upload.newSegsLeft[0];
|
|
226
|
+
(0, assert_1.assert)(!!segInfo);
|
|
227
|
+
const len = Math.min(maxSegs, segInfo.len);
|
|
228
|
+
const segs = await this.src.segSrc.read(len);
|
|
229
|
+
(0, assert_1.assert)(!!segs && (segs.length === len));
|
|
230
|
+
if (segInfo.len > len) {
|
|
231
|
+
upload.newSegsLeft.splice(1, 0, {
|
|
232
|
+
len: segInfo.len - len,
|
|
233
|
+
thisVerOfs: segInfo.thisVerOfs + len
|
|
234
|
+
});
|
|
267
235
|
}
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
if (
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
trans: uploadInfo.transactionId,
|
|
274
|
-
last: true
|
|
275
|
-
};
|
|
276
|
-
await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
|
|
277
|
-
uploadInfo.done = true;
|
|
278
|
-
uploadInfo.awaiting = undefined;
|
|
236
|
+
const ofs = segInfo.thisVerOfs;
|
|
237
|
+
const trans = upload.transactionId;
|
|
238
|
+
if (upload.newSegsLeft.length === 1) {
|
|
239
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
|
|
240
|
+
this.info.needUpload = undefined;
|
|
279
241
|
}
|
|
280
242
|
else {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
trans: uploadInfo.transactionId
|
|
284
|
-
};
|
|
285
|
-
await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
|
|
286
|
-
if (segs.length === section.len) {
|
|
287
|
-
uploadInfo.awaiting.segs.splice(0, 1);
|
|
288
|
-
}
|
|
289
|
-
else {
|
|
290
|
-
section.ofs += segs.length;
|
|
291
|
-
section.len -= segs.length;
|
|
292
|
-
}
|
|
293
|
-
this.addToWorker(this);
|
|
243
|
+
await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
|
|
244
|
+
upload.newSegsLeft.splice(0, 1);
|
|
294
245
|
}
|
|
295
246
|
}
|
|
247
|
+
cancel() {
|
|
248
|
+
// XXX
|
|
249
|
+
throw new Error("UploadTask.cancel() not implemented.");
|
|
250
|
+
}
|
|
296
251
|
}
|
|
297
|
-
Object.freeze(
|
|
298
|
-
Object.freeze(
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
252
|
+
Object.freeze(UploadTask.prototype);
|
|
253
|
+
Object.freeze(UploadTask);
|
|
254
|
+
async function wholeVerUpload(src, uploadHeader, createObj) {
|
|
255
|
+
const header = await expectedHeaderLen(src, uploadHeader);
|
|
256
|
+
const segsSize = (await src.segSrc.getSize()).size;
|
|
257
|
+
return {
|
|
258
|
+
type: 'ordered-whole',
|
|
259
|
+
createObj,
|
|
260
|
+
header,
|
|
261
|
+
segsOfs: 0,
|
|
262
|
+
segsLeft: segsSize
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
async function diffVerUpload(src, uploadHeader, diff, newSegsPackOrder) {
|
|
266
|
+
const header = await expectedHeaderLen(src, uploadHeader);
|
|
267
|
+
return {
|
|
268
|
+
type: 'ordered-diff',
|
|
269
|
+
diff,
|
|
270
|
+
newSegsLeft: newSegsPackOrder,
|
|
271
|
+
header
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
async function expectedHeaderLen(src, uploadHeader) {
|
|
275
|
+
return (uploadHeader ?
|
|
276
|
+
uploadHeader.length : (await src.readHeader()).length);
|
|
277
|
+
}
|
|
278
|
+
function uploadSize(info) {
|
|
279
|
+
if (info.type === 'ordered-whole') {
|
|
280
|
+
return info.header + info.segsLeft;
|
|
309
281
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
});
|
|
282
|
+
else if (info.type === 'ordered-diff') {
|
|
283
|
+
let uploadSize = info.header;
|
|
284
|
+
for (const { len } of info.newSegsLeft) {
|
|
285
|
+
uploadSize += len;
|
|
286
|
+
}
|
|
287
|
+
return uploadSize;
|
|
317
288
|
}
|
|
318
|
-
|
|
319
|
-
|
|
289
|
+
else {
|
|
290
|
+
throw new Error(`Unimplemented upload type ${info.type}`);
|
|
320
291
|
}
|
|
321
292
|
}
|
|
322
|
-
|
|
323
|
-
|
|
293
|
+
function executorLabelFor(info) {
|
|
294
|
+
return ((uploadSize(info) <= MAX_FAST_UPLOAD) ? 'fast' : 'long');
|
|
295
|
+
}
|
|
296
|
+
function noop() { }
|
|
324
297
|
Object.freeze(exports);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
declare type WritableFS = web3n.files.WritableFS;
|
|
2
|
+
export declare class AppDataFolders {
|
|
3
|
+
private readonly fs;
|
|
4
|
+
private writingSync;
|
|
5
|
+
private syncFolderProc;
|
|
6
|
+
private constructor();
|
|
7
|
+
static make(rootFS: WritableFS): Promise<AppDataFolders>;
|
|
8
|
+
getOrMake(folder: string): Promise<WritableFS>;
|
|
9
|
+
private syncP;
|
|
10
|
+
private makeSyncedFolder;
|
|
11
|
+
private syncBeforeChange;
|
|
12
|
+
private uploadAfterCreationOf;
|
|
13
|
+
private startSyncProc;
|
|
14
|
+
stopSync(): void;
|
|
15
|
+
}
|
|
16
|
+
export {};
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2022 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.AppDataFolders = void 0;
|
|
20
|
+
const rxjs_1 = require("rxjs");
|
|
21
|
+
const operators_1 = require("rxjs/operators");
|
|
22
|
+
const _1 = require(".");
|
|
23
|
+
const assert_1 = require("../../../lib-common/assert");
|
|
24
|
+
const synced_1 = require("../../../lib-common/processes/synced");
|
|
25
|
+
class AppDataFolders {
|
|
26
|
+
constructor(fs) {
|
|
27
|
+
var _a;
|
|
28
|
+
this.fs = fs;
|
|
29
|
+
this.writingSync = undefined;
|
|
30
|
+
(0, assert_1.assert)(!!((_a = this.fs.v) === null || _a === void 0 ? void 0 : _a.sync));
|
|
31
|
+
this.startSyncProc();
|
|
32
|
+
Object.seal(this);
|
|
33
|
+
}
|
|
34
|
+
static async make(rootFS) {
|
|
35
|
+
const fs = await rootFS.writableSubRoot(_1.sysFolders.appData, { create: false });
|
|
36
|
+
return new AppDataFolders(fs);
|
|
37
|
+
}
|
|
38
|
+
async getOrMake(folder) {
|
|
39
|
+
var _a;
|
|
40
|
+
if ((_a = this.fs.v) === null || _a === void 0 ? void 0 : _a.sync) {
|
|
41
|
+
try {
|
|
42
|
+
return await this.fs.writableSubRoot(folder, { create: false });
|
|
43
|
+
}
|
|
44
|
+
catch (exc) {
|
|
45
|
+
if ((exc.type === 'file')
|
|
46
|
+
&& exc.notFound) {
|
|
47
|
+
return this.makeSyncedFolder(folder);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
throw exc;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
return await this.fs.writableSubRoot(folder);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
syncP() {
|
|
59
|
+
if (!this.writingSync) {
|
|
60
|
+
this.writingSync = new synced_1.SingleProc();
|
|
61
|
+
}
|
|
62
|
+
return this.writingSync;
|
|
63
|
+
}
|
|
64
|
+
makeSyncedFolder(folder) {
|
|
65
|
+
return this.syncP().startOrChain(async () => {
|
|
66
|
+
await this.syncBeforeChange();
|
|
67
|
+
try {
|
|
68
|
+
await this.fs.makeFolder(folder, true);
|
|
69
|
+
}
|
|
70
|
+
catch (exc) {
|
|
71
|
+
if ((exc.type === 'file')
|
|
72
|
+
&& exc.alreadyExists) {
|
|
73
|
+
return this.fs.writableSubRoot(folder, { create: false });
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
throw exc;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const fs = await this.fs.writableSubRoot(folder, { create: false });
|
|
80
|
+
await this.uploadAfterCreationOf(folder);
|
|
81
|
+
return fs;
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
async syncBeforeChange() {
|
|
85
|
+
const { state } = await this.fs.v.sync.updateStatusInfo('');
|
|
86
|
+
if (state === 'behind') {
|
|
87
|
+
await this.fs.v.sync.adoptRemote('');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async uploadAfterCreationOf(folder) {
|
|
91
|
+
// XXX must add work with not-online condition
|
|
92
|
+
await this.fs.v.sync.upload(folder);
|
|
93
|
+
await this.fs.v.sync.upload('');
|
|
94
|
+
}
|
|
95
|
+
startSyncProc() {
|
|
96
|
+
this.syncFolderProc = (new rxjs_1.Observable((obs) => this.fs.watchFolder('', obs)))
|
|
97
|
+
.pipe((0, operators_1.filter)(ev => (ev.type === 'remote-change')), (0, operators_1.mergeMap)(() => this.syncP().startOrChain(() => this.fs.v.sync.adoptRemote('')), 1))
|
|
98
|
+
.subscribe();
|
|
99
|
+
}
|
|
100
|
+
stopSync() {
|
|
101
|
+
if (this.syncFolderProc) {
|
|
102
|
+
this.syncFolderProc.unsubscribe();
|
|
103
|
+
this.syncFolderProc = undefined;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
exports.AppDataFolders = AppDataFolders;
|
|
108
|
+
Object.freeze(AppDataFolders.prototype);
|
|
109
|
+
Object.freeze(AppDataFolders);
|
|
110
|
+
Object.freeze(exports);
|