core-3nweb-client-lib 0.25.6 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +2 -2
  2. package/build/api-defs/asmail.d.ts +1 -1
  3. package/build/api-defs/files.d.ts +282 -70
  4. package/build/core/app-files.js +7 -7
  5. package/build/core/asmail/config/common.js +4 -4
  6. package/build/core/asmail/config/index.js +2 -2
  7. package/build/core/asmail/config/published-intro-key.js +1 -1
  8. package/build/core/asmail/delivery/common.js +7 -7
  9. package/build/core/asmail/delivery/index.js +7 -6
  10. package/build/core/asmail/delivery/msg.js +8 -7
  11. package/build/core/asmail/delivery/per-recipient-wip.js +3 -3
  12. package/build/core/asmail/inbox/attachments/fs.js +11 -1
  13. package/build/core/asmail/inbox/cached-msgs.js +3 -3
  14. package/build/core/asmail/inbox/inbox-events.js +5 -4
  15. package/build/core/asmail/inbox/index.js +12 -12
  16. package/build/core/asmail/inbox/msg-downloader.js +3 -3
  17. package/build/core/asmail/inbox/msg-indexing.js +4 -4
  18. package/build/core/asmail/inbox/msg-on-disk.js +7 -7
  19. package/build/core/asmail/index.d.ts +3 -3
  20. package/build/core/asmail/index.js +13 -8
  21. package/build/core/asmail/key-verification.js +5 -5
  22. package/build/core/asmail/keyring/common.js +7 -6
  23. package/build/core/asmail/keyring/correspondent-keys.js +8 -7
  24. package/build/core/asmail/keyring/id-to-email-map.js +2 -1
  25. package/build/core/asmail/keyring/index.d.ts +7 -8
  26. package/build/core/asmail/keyring/index.js +15 -14
  27. package/build/core/asmail/keyring/keyring-storage.js +4 -3
  28. package/build/core/asmail/msg/opener.js +3 -3
  29. package/build/core/asmail/msg/packer.js +13 -13
  30. package/build/core/asmail/sending-params/own-params.js +4 -4
  31. package/build/core/asmail/sending-params/params-from-others.js +3 -3
  32. package/build/core/id-manager.js +8 -5
  33. package/build/core/index.d.ts +2 -1
  34. package/build/core/index.js +14 -14
  35. package/build/core/sign-in.d.ts +5 -4
  36. package/build/core/sign-in.js +12 -14
  37. package/build/core/sign-up.d.ts +1 -0
  38. package/build/core/sign-up.js +15 -11
  39. package/build/core/storage/common/json-saving.d.ts +21 -0
  40. package/build/core/storage/common/json-saving.js +82 -0
  41. package/build/core/storage/common/obj-info-file.d.ts +51 -0
  42. package/build/core/storage/common/obj-info-file.js +153 -5
  43. package/build/core/storage/common/utils.d.ts +2 -0
  44. package/build/core/storage/common/utils.js +32 -0
  45. package/build/core/storage/index.d.ts +3 -17
  46. package/build/core/storage/index.js +57 -77
  47. package/build/core/storage/local/obj-files-gc.d.ts +2 -0
  48. package/build/core/storage/local/obj-files-gc.js +53 -39
  49. package/build/core/storage/local/obj-files.d.ts +6 -9
  50. package/build/core/storage/local/obj-files.js +16 -19
  51. package/build/core/storage/local/obj-status.d.ts +20 -30
  52. package/build/core/storage/local/obj-status.js +46 -113
  53. package/build/core/storage/local/storage.d.ts +15 -5
  54. package/build/core/storage/local/storage.js +37 -18
  55. package/build/core/storage/synced/downloader.js +7 -6
  56. package/build/core/storage/synced/obj-files-gc.d.ts +6 -1
  57. package/build/core/storage/synced/obj-files-gc.js +106 -13
  58. package/build/core/storage/synced/obj-files.d.ts +46 -47
  59. package/build/core/storage/synced/obj-files.js +207 -154
  60. package/build/core/storage/synced/obj-status.d.ts +103 -42
  61. package/build/core/storage/synced/obj-status.js +525 -137
  62. package/build/core/storage/synced/remote-events.d.ts +11 -12
  63. package/build/core/storage/synced/remote-events.js +80 -57
  64. package/build/core/storage/synced/storage.d.ts +24 -5
  65. package/build/core/storage/synced/storage.js +123 -38
  66. package/build/core/storage/synced/upload-header-file.d.ts +4 -0
  67. package/build/core/storage/synced/upload-header-file.js +64 -0
  68. package/build/core/storage/synced/upsyncer.d.ts +15 -9
  69. package/build/core/storage/synced/upsyncer.js +219 -246
  70. package/build/core/storage/system-folders/apps-data.d.ts +16 -0
  71. package/build/core/storage/system-folders/apps-data.js +110 -0
  72. package/build/core/storage/system-folders/index.d.ts +18 -0
  73. package/build/core/storage/system-folders/index.js +77 -0
  74. package/build/core-ipc/common-caps.js +3 -3
  75. package/build/core-ipc/generic.js +8 -8
  76. package/build/core-ipc/startup-caps.js +2 -2
  77. package/build/cryptors.js +6 -2
  78. package/build/ipc-via-protobuf/asmail-cap.js +67 -83
  79. package/build/ipc-via-protobuf/bytes.js +16 -17
  80. package/build/ipc-via-protobuf/connector-clients-side.d.ts +3 -0
  81. package/build/ipc-via-protobuf/connector-clients-side.js +62 -25
  82. package/build/ipc-via-protobuf/connector-services-side.js +10 -10
  83. package/build/ipc-via-protobuf/connector.js +4 -4
  84. package/build/ipc-via-protobuf/file.d.ts +48 -12
  85. package/build/ipc-via-protobuf/file.js +476 -120
  86. package/build/ipc-via-protobuf/fs.d.ts +8 -0
  87. package/build/ipc-via-protobuf/fs.js +592 -159
  88. package/build/ipc-via-protobuf/log-cap.js +2 -2
  89. package/build/ipc-via-protobuf/mailerid.js +3 -3
  90. package/build/ipc-via-protobuf/protobuf-msg.d.ts +1 -0
  91. package/build/ipc-via-protobuf/protobuf-msg.js +11 -7
  92. package/build/ipc-via-protobuf/startup-cap.js +23 -23
  93. package/build/ipc-via-protobuf/storage-cap.js +12 -12
  94. package/build/ipc.js +7 -2
  95. package/build/lib-client/3nstorage/exceptions.d.ts +12 -8
  96. package/build/lib-client/3nstorage/exceptions.js +31 -10
  97. package/build/lib-client/3nstorage/service.d.ts +16 -2
  98. package/build/lib-client/3nstorage/service.js +109 -39
  99. package/build/lib-client/3nstorage/util/file-based-json.d.ts +2 -1
  100. package/build/lib-client/3nstorage/util/file-based-json.js +1 -1
  101. package/build/lib-client/3nstorage/xsp-fs/attrs.js +17 -17
  102. package/build/lib-client/3nstorage/xsp-fs/common.d.ts +52 -14
  103. package/build/lib-client/3nstorage/xsp-fs/common.js +31 -16
  104. package/build/lib-client/3nstorage/xsp-fs/file-node.d.ts +1 -0
  105. package/build/lib-client/3nstorage/xsp-fs/file-node.js +18 -14
  106. package/build/lib-client/3nstorage/xsp-fs/file.d.ts +31 -6
  107. package/build/lib-client/3nstorage/xsp-fs/file.js +74 -23
  108. package/build/lib-client/3nstorage/xsp-fs/folder-node-serialization.js +4 -4
  109. package/build/lib-client/3nstorage/xsp-fs/folder-node.d.ts +24 -11
  110. package/build/lib-client/3nstorage/xsp-fs/folder-node.js +599 -189
  111. package/build/lib-client/3nstorage/xsp-fs/fs.d.ts +45 -9
  112. package/build/lib-client/3nstorage/xsp-fs/fs.js +326 -74
  113. package/build/lib-client/3nstorage/xsp-fs/link-node.d.ts +1 -0
  114. package/build/lib-client/3nstorage/xsp-fs/link-node.js +7 -2
  115. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.d.ts +30 -20
  116. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.js +239 -106
  117. package/build/lib-client/3nstorage/xsp-fs/node-persistence.d.ts +1 -1
  118. package/build/lib-client/3nstorage/xsp-fs/node-persistence.js +18 -19
  119. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v1.js +5 -5
  120. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v2.js +56 -56
  121. package/build/lib-client/3nweb-signup.js +4 -4
  122. package/build/lib-client/asmail/recipient.js +15 -15
  123. package/build/lib-client/asmail/sender.js +22 -22
  124. package/build/lib-client/asmail/service-config.js +3 -3
  125. package/build/lib-client/cryptor/cryptor-in-worker.js +19 -17
  126. package/build/lib-client/cryptor/cryptor-wasm.js +1 -1
  127. package/build/lib-client/cryptor/cryptor.js +4 -2
  128. package/build/lib-client/cryptor/cryptor.wasm +0 -0
  129. package/build/lib-client/cryptor/in-proc-js.js +1 -1
  130. package/build/lib-client/cryptor/in-proc-wasm.js +7 -7
  131. package/build/lib-client/cryptor/worker-js.js +2 -2
  132. package/build/lib-client/cryptor/worker-wasm.js +2 -2
  133. package/build/lib-client/files-select.js +1 -1
  134. package/build/lib-client/files.d.ts +1 -1
  135. package/build/lib-client/files.js +71 -4
  136. package/build/lib-client/fs-collection.js +3 -2
  137. package/build/lib-client/fs-sync-utils.d.ts +5 -0
  138. package/build/lib-client/fs-sync-utils.js +61 -0
  139. package/build/lib-client/fs-view.d.ts +14 -0
  140. package/build/lib-client/fs-view.js +33 -0
  141. package/build/lib-client/key-derivation.js +1 -1
  142. package/build/lib-client/local-files/dev-file-sink.js +9 -9
  143. package/build/lib-client/local-files/dev-file-src.js +2 -2
  144. package/build/lib-client/local-files/device-fs.d.ts +1 -1
  145. package/build/lib-client/local-files/device-fs.js +56 -54
  146. package/build/lib-client/logging/log-to-file.d.ts +1 -1
  147. package/build/lib-client/logging/log-to-file.js +11 -11
  148. package/build/lib-client/mailer-id/login.js +7 -7
  149. package/build/lib-client/mailer-id/provisioner.js +12 -12
  150. package/build/lib-client/objs-on-disk/file-writing-proc.js +5 -5
  151. package/build/lib-client/objs-on-disk/obj-folders.js +33 -33
  152. package/build/lib-client/objs-on-disk/obj-on-disk.d.ts +13 -2
  153. package/build/lib-client/objs-on-disk/obj-on-disk.js +24 -9
  154. package/build/lib-client/request-utils.d.ts +1 -0
  155. package/build/lib-client/request-utils.js +14 -14
  156. package/build/lib-client/server-events.d.ts +3 -3
  157. package/build/lib-client/server-events.js +12 -10
  158. package/build/lib-client/service-locator.js +10 -10
  159. package/build/lib-client/user-with-mid-session.js +7 -7
  160. package/build/lib-client/user-with-pkl-session.js +25 -25
  161. package/build/lib-client/ws-utils.js +3 -3
  162. package/build/lib-common/async-cryptor-wrap.js +4 -4
  163. package/build/lib-common/async-fs-node.d.ts +5 -3
  164. package/build/lib-common/async-fs-node.js +19 -18
  165. package/build/lib-common/byte-streaming/pipe.js +1 -1
  166. package/build/lib-common/byte-streaming/wrapping.js +17 -17
  167. package/build/lib-common/canonical-address.js +1 -1
  168. package/build/lib-common/exceptions/error.d.ts +1 -0
  169. package/build/lib-common/exceptions/error.js +7 -6
  170. package/build/lib-common/exceptions/file.js +10 -1
  171. package/build/lib-common/ipc/generic-ipc.js +2 -2
  172. package/build/lib-common/ipc/ws-ipc.js +2 -2
  173. package/build/lib-common/json-utils.js +2 -1
  174. package/build/lib-common/mid-sigs-NaCl-Ed.js +14 -14
  175. package/build/lib-common/objs-on-disk/file-layout.d.ts +19 -0
  176. package/build/lib-common/objs-on-disk/file-layout.js +130 -12
  177. package/build/lib-common/objs-on-disk/obj-file.d.ts +13 -2
  178. package/build/lib-common/objs-on-disk/obj-file.js +99 -37
  179. package/build/lib-common/objs-on-disk/utils.d.ts +1 -0
  180. package/build/lib-common/objs-on-disk/utils.js +4 -4
  181. package/build/lib-common/objs-on-disk/v1-obj-file-format.js +14 -14
  182. package/build/lib-common/processes/deferred.d.ts +6 -0
  183. package/build/lib-common/processes/deferred.js +30 -0
  184. package/build/lib-common/processes/labelled-exec-pools.d.ts +33 -0
  185. package/build/lib-common/processes/labelled-exec-pools.js +141 -0
  186. package/build/lib-common/processes/pressure.d.ts +7 -0
  187. package/build/lib-common/processes/pressure.js +56 -0
  188. package/build/lib-common/processes/sleep.d.ts +1 -0
  189. package/build/lib-common/processes/sleep.js +26 -0
  190. package/build/lib-common/{processes.d.ts → processes/synced.d.ts} +0 -40
  191. package/build/lib-common/{processes.js → processes/synced.js} +187 -204
  192. package/build/lib-common/processes/timeout.d.ts +1 -0
  193. package/build/lib-common/processes/timeout.js +51 -0
  194. package/build/lib-common/random-node.js +7 -7
  195. package/build/lib-common/service-api/3nstorage/owner.d.ts +100 -39
  196. package/build/lib-common/service-api/3nstorage/owner.js +85 -42
  197. package/build/lib-common/service-api/asmail/delivery.js +2 -2
  198. package/build/lib-common/service-api/asmail/retrieval.js +1 -1
  199. package/build/lib-common/timed-cache.d.ts +1 -0
  200. package/build/lib-common/timed-non-weak-cache.d.ts +1 -0
  201. package/build/lib-common/timed-non-weak-cache.js +11 -0
  202. package/build/lib-common/utils-for-observables.d.ts +15 -1
  203. package/build/lib-common/utils-for-observables.js +70 -19
  204. package/build/lib-common/weak-cache.d.ts +1 -0
  205. package/build/lib-common/weak-cache.js +12 -1
  206. package/build/lib-index.d.ts +2 -1
  207. package/build/lib-index.js +10 -7
  208. package/build/protos/asmail.proto.js +12912 -7127
  209. package/build/protos/file.proto.js +4848 -2399
  210. package/build/protos/fs.proto.js +9230 -3445
  211. package/package.json +8 -7
  212. package/protos/file.proto +91 -11
  213. package/protos/fs.proto +107 -8
  214. package/build/core/storage/synced/upsync-status.d.ts +0 -41
  215. package/build/core/storage/synced/upsync-status.js +0 -158
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  /*
3
- Copyright (C) 2020 3NSoft Inc.
3
+ Copyright (C) 2020, 2022 3NSoft Inc.
4
4
 
5
5
  This program is free software: you can redistribute it and/or modify it under
6
6
  the terms of the GNU General Public License as published by the Free Software
@@ -13,312 +13,285 @@
13
13
  See the GNU General Public License for more details.
14
14
 
15
15
  You should have received a copy of the GNU General Public License along with
16
- this program. If not, see <http://www.gnu.org/licenses/>. */
16
+ this program. If not, see <http://www.gnu.org/licenses/>.
17
+ */
17
18
  Object.defineProperty(exports, "__esModule", { value: true });
18
19
  exports.UpSyncer = void 0;
19
- const operators_1 = require("rxjs/operators");
20
- const processes_1 = require("../../../lib-common/processes");
21
- const upsync_status_1 = require("./upsync-status");
20
+ const labelled_exec_pools_1 = require("../../../lib-common/processes/labelled-exec-pools");
21
+ const exceptions_1 = require("../../../lib-client/3nstorage/exceptions");
22
22
  const assert_1 = require("../../../lib-common/assert");
23
+ const deferred_1 = require("../../../lib-common/processes/deferred");
24
+ const buffer_utils_1 = require("../../../lib-common/buffer-utils");
23
25
  const MAX_CHUNK_SIZE = 512 * 1024;
26
+ const MAX_FAST_UPLOAD = 2 * 1024 * 1024;
24
27
  class UpSyncer {
25
- constructor(remoteStorage, files, logError) {
28
+ constructor(remoteStorage, logError) {
26
29
  this.remoteStorage = remoteStorage;
27
- this.files = files;
28
30
  this.logError = logError;
29
- this.uploads = new Map();
30
- this.addToWorker = (ready) => {
31
- this.worker.add(ready);
32
- };
33
- this.worker = new processes_1.Worker(async (u) => {
34
- await u.process();
35
- if (u.isDone() && (u === this.uploads.get(u.obj))) {
36
- this.uploads.delete(u.obj);
37
- }
38
- }, async (queue) => {
39
- for (const u of queue) {
40
- u.stop();
41
- this.uploads.delete(u.obj);
42
- }
43
- });
31
+ this.execPools = new labelled_exec_pools_1.LabelledExecPools([
32
+ { label: 'long', maxProcs: 1 },
33
+ { label: 'fast', maxProcs: 1 }
34
+ ], logError);
44
35
  Object.seal(this);
45
36
  }
46
- getOrMakeUploadsFor(obj) {
47
- let uploads = this.uploads.get(obj);
48
- if (!uploads) {
49
- const status = new upsync_status_1.UpSyncTasks(obj.objFolder, this.logError);
50
- uploads = new ObjUpSync(obj, status, this.remoteStorage, this.addToWorker, this.logError);
51
- this.uploads.set(obj, uploads);
52
- }
53
- return uploads;
54
- }
55
37
  start() {
56
- this.worker.start(2);
38
+ this.execPools.start();
57
39
  }
58
40
  async stop() {
59
- await this.worker.stop();
60
- for (const upload of this.uploads.values()) {
61
- upload.stop();
62
- }
63
- this.uploads.clear();
64
- }
65
- get chunkSize() {
66
- return (this.remoteStorage.maxChunkSize ?
67
- Math.min(this.remoteStorage.maxChunkSize, MAX_CHUNK_SIZE) :
68
- MAX_CHUNK_SIZE);
41
+ await this.execPools.stop(); // implicitly cancels all upsync tasks
69
42
  }
43
+ /**
44
+ * Creates an rxjs operator to tap saving process, starting upload while
45
+ * writing is ongoing.
46
+ */
70
47
  tapFileWrite(obj, isNew, newVersion, baseVersion) {
71
- const objUploads = this.getOrMakeUploadsFor(obj);
72
- return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
48
+ throw new Error('UpSyncer.tapFileWrite() not implemented');
49
+ // const objUploads = this.getOrMakeUploadsFor(obj);
50
+ // return objUploads.tapFileWrite(isNew, newVersion, baseVersion);
73
51
  }
74
52
  async removeCurrentVersionOf(obj) {
75
- const objUploads = this.getOrMakeUploadsFor(obj);
76
- objUploads.removeCurrentVersion();
53
+ throw new Error('UpSyncer.removeCurrentVersionOf() not implemented');
54
+ // const objUploads = this.getOrMakeUploadsFor(obj);
55
+ // XXX is removal set in this older code? This may need to change anyway
56
+ // if (objUploads.neededExecutor()) {
57
+ // this.execPools.add(objUploads);
58
+ // }
59
+ }
60
+ async uploadFromDisk(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote) {
61
+ const task = await UploadTask.for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createOnRemote, this.remoteStorage, this.execPools);
62
+ this.execPools.add(task);
63
+ await task.completion();
77
64
  }
78
65
  }
79
66
  exports.UpSyncer = UpSyncer;
80
67
  Object.freeze(UpSyncer.prototype);
81
68
  Object.freeze(UpSyncer);
82
- class ObjUpSync {
83
- constructor(obj, status, remoteStorage, addToWorker, logError) {
84
- this.obj = obj;
85
- this.status = status;
69
+ class UploadTask {
70
+ constructor(remoteStorage, objId, objStatus, src, execPools, info, uploadHeader) {
86
71
  this.remoteStorage = remoteStorage;
87
- this.addToWorker = addToWorker;
88
- this.logError = logError;
89
- this.objWriteTaps = new Map();
90
- Object.freeze(this);
91
- }
92
- tapFileWrite(isObjNew, newVersion, baseVersion) {
93
- const tap = new ObjWriteTap(this.obj, newVersion, baseVersion, isObjNew);
94
- const uploadInfo = upsync_status_1.makeUploadInfo(tap.version, tap.baseVersion);
95
- this.objWriteTaps.set(uploadInfo, tap);
96
- return tap.tapOperator();
97
- }
98
- stop() {
99
- // XXX
100
- // - is something needed here?
72
+ this.objId = objId;
73
+ this.objStatus = objStatus;
74
+ this.src = src;
75
+ this.execPools = execPools;
76
+ this.info = info;
77
+ this.uploadHeader = uploadHeader;
78
+ this.uploadCompletion = (0, deferred_1.defer)();
79
+ this.execLabel = executorLabelFor(this.info.needUpload);
80
+ Object.seal(this);
101
81
  }
102
- removeCurrentVersion() {
103
- this.status.queueTask({
104
- type: 'removal',
105
- currentVersion: true
106
- });
107
- this.addToWorker(this);
82
+ static async for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createObj, remoteStorage, execPools) {
83
+ const src = await obj.getObjSrcFromLocalAndSyncedBranch(localVersion);
84
+ let needUpload;
85
+ if (syncedBase) {
86
+ const { diff, newSegsPackOrder } = await obj.diffForUploadOf(localVersion);
87
+ needUpload = await diffVerUpload(src, uploadHeader, diff, newSegsPackOrder);
88
+ }
89
+ else {
90
+ needUpload = await wholeVerUpload(src, uploadHeader, createObj);
91
+ }
92
+ const info = {
93
+ localVersion,
94
+ uploadVersion,
95
+ baseVersion: syncedBase,
96
+ needUpload
97
+ };
98
+ const objStatus = obj.statusObj();
99
+ await objStatus.recordUploadStart(info);
100
+ return new UploadTask(remoteStorage, obj.objId, objStatus, src, execPools, info, uploadHeader);
108
101
  }
109
- removeArchivedVersion(version) {
110
- this.status.queueTask({
111
- type: 'removal',
112
- archivedVersions: version
113
- });
114
- this.addToWorker(this);
102
+ neededExecutor() {
103
+ return (!this.info.needUpload ? undefined : this.execLabel);
115
104
  }
116
- isDone() {
117
- return this.status.isDone();
105
+ completion() {
106
+ return this.uploadCompletion.promise;
118
107
  }
119
108
  async process() {
120
- if (this.isDone()) {
109
+ if (!this.info.needUpload) {
121
110
  return;
122
111
  }
123
112
  try {
124
- const task = (await this.status.nextTask());
125
- if (task.type === 'upload') {
126
- await this.processUpload(task);
113
+ const upload = this.info.needUpload;
114
+ if (upload.type === 'ordered-diff') {
115
+ if (upload.transactionId) {
116
+ await this.continueOrderedDiffUpload(upload);
117
+ }
118
+ else {
119
+ await this.startOrderedDiffUpload(upload);
120
+ }
127
121
  }
128
- else if (task.type === 'removal') {
129
- await this.processRemoval(task);
130
- }
131
- else if (task.type === 'archiving') {
132
- await this.processArchival(task);
122
+ else if (upload.type === 'ordered-whole') {
123
+ if (upload.transactionId) {
124
+ await this.continueOrderedUpload(upload);
125
+ }
126
+ else {
127
+ await this.startOrderedUpload(upload);
128
+ }
133
129
  }
134
130
  else {
135
- throw new Error(`This shouldn't be reached`);
131
+ throw new Error(`Unimplemented ${upload.type} upload`);
136
132
  }
137
- }
138
- catch (err) {
139
- await this.logError(err, `Error occured in uploader processing`);
140
- }
141
- if (!this.isDone()) {
142
- this.addToWorker(this);
143
- }
144
- }
145
- async processArchival(task) {
146
- // XXX tell server to archive current version
147
- await this.logError(`Archival of current version is not implemented, yet.`);
148
- await this.status.recordTaskCompletion(task);
149
- }
150
- async processRemoval(task) {
151
- if (task.currentVersion) {
152
- if (this.obj.objId) {
153
- await this.remoteStorage.deleteObj(this.obj.objId);
133
+ await this.objStatus.recordUploadInterimState(this.info);
134
+ if (this.info.needUpload) {
135
+ this.execPools.add(this);
154
136
  }
155
137
  else {
156
- await this.logError(`Root obj can't be removed`);
138
+ this.uploadCompletion.resolve();
157
139
  }
158
140
  }
159
- if (task.archivedVersions) {
160
- // XXX tell server to remove given archived versions
161
- await this.logError(`Removal of archived version is not implemented, yet.`);
141
+ catch (exc) {
142
+ this.info.needUpload = undefined;
143
+ this.uploadCompletion.reject((0, exceptions_1.makeFSSyncException)(`obj-upload`, {
144
+ message: `Fail to upload local version ${this.info.uploadVersion}`,
145
+ localVersion: this.info.uploadVersion,
146
+ cause: exc
147
+ }));
148
+ await this.objStatus.recordUploadCancellation(this.info);
162
149
  }
163
- await this.status.recordTaskCompletion(task);
164
150
  }
165
- async processUpload(task) {
166
- const tap = this.objWriteTaps.get(task);
167
- if (tap && !task.done) {
168
- await this.uploadReadyBytes(task, tap);
151
+ async startOrderedUpload(upload) {
152
+ const maxSegs = this.maxUploadChunk() - upload.header;
153
+ (0, assert_1.assert)(maxSegs > 1);
154
+ const segsToUpload = Math.min(upload.segsLeft, maxSegs);
155
+ const header = await this.headerToUpload();
156
+ let segs = undefined;
157
+ if (segsToUpload > 0) {
158
+ await this.src.segSrc.seek(upload.segsOfs);
159
+ segs = await this.src.segSrc.read(segsToUpload);
169
160
  }
170
- if (tap && !task.done) {
171
- await this.status.recordInterimStateOfCurrentTask(task);
161
+ (0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
162
+ const ver = this.info.uploadVersion;
163
+ const create = (upload.createObj ? true : undefined);
164
+ if (segsToUpload === upload.segsLeft) {
165
+ await this.remoteStorage.saveNewObjVersion(this.objId, { ver, create, last: true }, undefined, { header, segs });
166
+ this.info.needUpload = undefined;
172
167
  }
173
168
  else {
174
- this.objWriteTaps.delete(task);
175
- await this.status.recordTaskCompletion(task);
176
- }
177
- }
178
- async uploadReadyBytes(uploadInfo, tap) {
179
- if (tap.cancelledOrErred()) {
180
- await this.cancelTransactionIfOpen(uploadInfo);
181
- return;
182
- }
183
- if (uploadInfo.transactionId) {
184
- await this.doUploadInTransaction(uploadInfo, tap);
185
- }
186
- else if (tap.isDone) {
187
- // XXX current code does upload after file is written, future code
188
- // can start upload sooner, and do diff-ed uploads
189
- await this.doFirstUpload(uploadInfo, tap);
169
+ upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver, create }, undefined, { header, segs });
170
+ if (!upload.transactionId) {
171
+ // XXX should this be runtime exception saying that remote acts badly ?
172
+ throw new Error(`Server didn't start obj saving transaction`);
173
+ }
174
+ upload.header = undefined;
175
+ upload.segsOfs += segsToUpload;
176
+ upload.segsLeft -= segsToUpload;
190
177
  }
191
178
  }
192
- async cancelTransactionIfOpen(uploadInfo) {
193
- const txnId = uploadInfo.transactionId;
194
- if (!txnId) {
195
- return;
196
- }
197
- await this.remoteStorage.cancelTransaction(this.obj.objId, txnId)
198
- .catch((exc) => {
199
- if (!exc.unknownTransaction) {
200
- throw exc;
201
- }
202
- });
179
+ async headerToUpload() {
180
+ return (this.uploadHeader ?
181
+ this.uploadHeader : await this.src.readHeader());
203
182
  }
204
- async doFirstUpload(uploadInfo, tap) {
205
- assert_1.assert(tap.isDone, `Current implementation works only on completely saved to disk objects`);
206
- assert_1.assert(!uploadInfo.awaiting);
207
- const src = await this.obj.getObjSrc(uploadInfo.version);
208
- // note that endlessness/finiteness of src is not used
209
- const header = await src.readHeader();
210
- const maxChunkLen = (this.remoteStorage.maxChunkSize ?
211
- this.remoteStorage.maxChunkSize - header.length : MAX_CHUNK_SIZE);
212
- assert_1.assert(maxChunkLen > 0);
213
- const segs = await src.segSrc.read(maxChunkLen);
214
- const srcLen = (await src.segSrc.getSize()).size;
215
- let last;
216
- let bytesToSend;
217
- if (segs) {
218
- if (segs.length === maxChunkLen) {
219
- last = (srcLen === segs.length);
220
- }
221
- else {
222
- last = false;
223
- }
224
- bytesToSend = [header, segs];
183
+ async startOrderedDiffUpload(upload) {
184
+ const diff = buffer_utils_1.utf8.pack(JSON.stringify(upload.diff));
185
+ const maxSegs = this.maxUploadChunk() - upload.header - diff.length;
186
+ (0, assert_1.assert)(maxSegs > 1);
187
+ const header = await this.headerToUpload();
188
+ const ver = this.info.uploadVersion;
189
+ if (upload.newSegsLeft.length === 0) {
190
+ await this.remoteStorage.saveNewObjVersion(this.objId, { ver, last: true }, undefined, { header, diff });
191
+ this.info.needUpload = undefined;
225
192
  }
226
193
  else {
227
- last = true;
228
- bytesToSend = header;
194
+ upload.transactionId = await this.remoteStorage.saveNewObjVersion(this.objId, { ver }, undefined, { header, diff });
195
+ if (!upload.transactionId) {
196
+ // XXX should this be runtime exception saying that remote acts badly ?
197
+ throw new Error(`Server didn't start obj saving transaction`);
198
+ }
199
+ upload.header = undefined;
229
200
  }
230
- if (last) {
231
- const opts = {
232
- header: header.length, ver: uploadInfo.version, last
233
- };
234
- await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytesToSend, opts, undefined);
235
- uploadInfo.done = true;
201
+ }
202
+ maxUploadChunk() {
203
+ return (this.remoteStorage.maxChunkSize ?
204
+ this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE);
205
+ }
206
+ async continueOrderedUpload(upload) {
207
+ const segsToUpload = Math.min(upload.segsLeft, this.maxUploadChunk());
208
+ await this.src.segSrc.seek(upload.segsOfs);
209
+ const segs = await this.src.segSrc.read(segsToUpload);
210
+ (0, assert_1.assert)(!!segs && (segs.length === segsToUpload));
211
+ const ofs = upload.segsOfs;
212
+ const trans = upload.transactionId;
213
+ if (segsToUpload === upload.segsLeft) {
214
+ await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
215
+ this.info.needUpload = undefined;
236
216
  }
237
217
  else {
238
- const opts = {
239
- header: header.length, ver: uploadInfo.version
240
- };
241
- const txnId = await this.remoteStorage.saveNewObjVersion(this.obj.objId, bytesToSend, opts, undefined);
242
- if (!txnId) {
243
- throw new Error(`Server didn't start obj saving transaction`);
244
- }
245
- uploadInfo.transactionId = txnId;
246
- uploadInfo.awaiting = {
247
- allByteOnDisk: true,
248
- segs: [{ ofs: segs.length, len: srcLen - segs.length }]
249
- };
250
- this.addToWorker(this);
218
+ await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
219
+ upload.segsOfs += segsToUpload;
220
+ upload.segsLeft -= segsToUpload;
251
221
  }
252
222
  }
253
- async doUploadInTransaction(uploadInfo, tap) {
254
- assert_1.assert(tap.isDone, `Current implementation works only on completely saved to disk objects`);
255
- assert_1.assert(!!uploadInfo.transactionId
256
- && !!uploadInfo.awaiting
257
- && (uploadInfo.awaiting.segs.length > 0)
258
- && (uploadInfo.awaiting.segs[0].len > 0));
259
- const section = uploadInfo.awaiting.segs[0];
260
- const lenToRead = Math.min(section.len, (this.remoteStorage.maxChunkSize ?
261
- this.remoteStorage.maxChunkSize : MAX_CHUNK_SIZE));
262
- const src = await this.obj.getObjSrc(uploadInfo.version);
263
- await src.segSrc.seek(section.ofs);
264
- const segs = await src.segSrc.read(lenToRead);
265
- if (!segs || (segs.length < lenToRead)) {
266
- throw new Error(`Unexpected end of obj source`);
223
+ async continueOrderedDiffUpload(upload) {
224
+ const maxSegs = this.maxUploadChunk();
225
+ const segInfo = upload.newSegsLeft[0];
226
+ (0, assert_1.assert)(!!segInfo);
227
+ const len = Math.min(maxSegs, segInfo.len);
228
+ const segs = await this.src.segSrc.read(len);
229
+ (0, assert_1.assert)(!!segs && (segs.length === len));
230
+ if (segInfo.len > len) {
231
+ upload.newSegsLeft.splice(1, 0, {
232
+ len: segInfo.len - len,
233
+ thisVerOfs: segInfo.thisVerOfs + len
234
+ });
267
235
  }
268
- const last = ((uploadInfo.awaiting.segs.length === 0) &&
269
- (segs.length === section.len));
270
- if (last) {
271
- const opts = {
272
- ofs: section.ofs,
273
- trans: uploadInfo.transactionId,
274
- last: true
275
- };
276
- await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
277
- uploadInfo.done = true;
278
- uploadInfo.awaiting = undefined;
236
+ const ofs = segInfo.thisVerOfs;
237
+ const trans = upload.transactionId;
238
+ if (upload.newSegsLeft.length === 1) {
239
+ await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans, last: true }, { segs });
240
+ this.info.needUpload = undefined;
279
241
  }
280
242
  else {
281
- const opts = {
282
- ofs: section.ofs,
283
- trans: uploadInfo.transactionId
284
- };
285
- await this.remoteStorage.saveNewObjVersion(this.obj.objId, segs, undefined, opts);
286
- if (segs.length === section.len) {
287
- uploadInfo.awaiting.segs.splice(0, 1);
288
- }
289
- else {
290
- section.ofs += segs.length;
291
- section.len -= segs.length;
292
- }
293
- this.addToWorker(this);
243
+ await this.remoteStorage.saveNewObjVersion(this.objId, undefined, { ofs, trans }, { segs });
244
+ upload.newSegsLeft.splice(0, 1);
294
245
  }
295
246
  }
247
+ cancel() {
248
+ // XXX
249
+ throw new Error("UploadTask.cancel() not implemented.");
250
+ }
296
251
  }
297
- Object.freeze(ObjUpSync.prototype);
298
- Object.freeze(ObjUpSync);
299
- class ObjWriteTap {
300
- constructor(obj, version, baseVersion, isObjNew) {
301
- this.obj = obj;
302
- this.version = version;
303
- this.baseVersion = baseVersion;
304
- this.isObjNew = isObjNew;
305
- this.isDone = false;
306
- this.isCancelled = false;
307
- this.err = undefined;
308
- Object.seal(this);
252
+ Object.freeze(UploadTask.prototype);
253
+ Object.freeze(UploadTask);
254
+ async function wholeVerUpload(src, uploadHeader, createObj) {
255
+ const header = await expectedHeaderLen(src, uploadHeader);
256
+ const segsSize = (await src.segSrc.getSize()).size;
257
+ return {
258
+ type: 'ordered-whole',
259
+ createObj,
260
+ header,
261
+ segsOfs: 0,
262
+ segsLeft: segsSize
263
+ };
264
+ }
265
+ async function diffVerUpload(src, uploadHeader, diff, newSegsPackOrder) {
266
+ const header = await expectedHeaderLen(src, uploadHeader);
267
+ return {
268
+ type: 'ordered-diff',
269
+ diff,
270
+ newSegsLeft: newSegsPackOrder,
271
+ header
272
+ };
273
+ }
274
+ async function expectedHeaderLen(src, uploadHeader) {
275
+ return (uploadHeader ?
276
+ uploadHeader.length : (await src.readHeader()).length);
277
+ }
278
+ function uploadSize(info) {
279
+ if (info.type === 'ordered-whole') {
280
+ return info.header + info.segsLeft;
309
281
  }
310
- tapOperator() {
311
- return operators_1.tap(w => { }, err => {
312
- this.err = err;
313
- this.isDone = true;
314
- }, () => {
315
- this.isDone = true;
316
- });
282
+ else if (info.type === 'ordered-diff') {
283
+ let uploadSize = info.header;
284
+ for (const { len } of info.newSegsLeft) {
285
+ uploadSize += len;
286
+ }
287
+ return uploadSize;
317
288
  }
318
- cancelledOrErred() {
319
- return (this.isCancelled || (this.isDone && this.err));
289
+ else {
290
+ throw new Error(`Unimplemented upload type ${info.type}`);
320
291
  }
321
292
  }
322
- Object.freeze(ObjWriteTap.prototype);
323
- Object.freeze(ObjWriteTap);
293
+ function executorLabelFor(info) {
294
+ return ((uploadSize(info) <= MAX_FAST_UPLOAD) ? 'fast' : 'long');
295
+ }
296
+ function noop() { }
324
297
  Object.freeze(exports);
@@ -0,0 +1,16 @@
1
+ declare type WritableFS = web3n.files.WritableFS;
2
+ export declare class AppDataFolders {
3
+ private readonly fs;
4
+ private writingSync;
5
+ private syncFolderProc;
6
+ private constructor();
7
+ static make(rootFS: WritableFS): Promise<AppDataFolders>;
8
+ getOrMake(folder: string): Promise<WritableFS>;
9
+ private syncP;
10
+ private makeSyncedFolder;
11
+ private syncBeforeChange;
12
+ private uploadAfterCreationOf;
13
+ private startSyncProc;
14
+ stopSync(): void;
15
+ }
16
+ export {};
@@ -0,0 +1,110 @@
1
+ "use strict";
2
+ /*
3
+ Copyright (C) 2022 3NSoft Inc.
4
+
5
+ This program is free software: you can redistribute it and/or modify it under
6
+ the terms of the GNU General Public License as published by the Free Software
7
+ Foundation, either version 3 of the License, or (at your option) any later
8
+ version.
9
+
10
+ This program is distributed in the hope that it will be useful, but
11
+ WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13
+ See the GNU General Public License for more details.
14
+
15
+ You should have received a copy of the GNU General Public License along with
16
+ this program. If not, see <http://www.gnu.org/licenses/>.
17
+ */
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ exports.AppDataFolders = void 0;
20
+ const rxjs_1 = require("rxjs");
21
+ const operators_1 = require("rxjs/operators");
22
+ const _1 = require(".");
23
+ const assert_1 = require("../../../lib-common/assert");
24
+ const synced_1 = require("../../../lib-common/processes/synced");
25
+ class AppDataFolders {
26
+ constructor(fs) {
27
+ var _a;
28
+ this.fs = fs;
29
+ this.writingSync = undefined;
30
+ (0, assert_1.assert)(!!((_a = this.fs.v) === null || _a === void 0 ? void 0 : _a.sync));
31
+ this.startSyncProc();
32
+ Object.seal(this);
33
+ }
34
+ static async make(rootFS) {
35
+ const fs = await rootFS.writableSubRoot(_1.sysFolders.appData, { create: false });
36
+ return new AppDataFolders(fs);
37
+ }
38
+ async getOrMake(folder) {
39
+ var _a;
40
+ if ((_a = this.fs.v) === null || _a === void 0 ? void 0 : _a.sync) {
41
+ try {
42
+ return await this.fs.writableSubRoot(folder, { create: false });
43
+ }
44
+ catch (exc) {
45
+ if ((exc.type === 'file')
46
+ && exc.notFound) {
47
+ return this.makeSyncedFolder(folder);
48
+ }
49
+ else {
50
+ throw exc;
51
+ }
52
+ }
53
+ }
54
+ else {
55
+ return await this.fs.writableSubRoot(folder);
56
+ }
57
+ }
58
+ syncP() {
59
+ if (!this.writingSync) {
60
+ this.writingSync = new synced_1.SingleProc();
61
+ }
62
+ return this.writingSync;
63
+ }
64
+ makeSyncedFolder(folder) {
65
+ return this.syncP().startOrChain(async () => {
66
+ await this.syncBeforeChange();
67
+ try {
68
+ await this.fs.makeFolder(folder, true);
69
+ }
70
+ catch (exc) {
71
+ if ((exc.type === 'file')
72
+ && exc.alreadyExists) {
73
+ return this.fs.writableSubRoot(folder, { create: false });
74
+ }
75
+ else {
76
+ throw exc;
77
+ }
78
+ }
79
+ const fs = await this.fs.writableSubRoot(folder, { create: false });
80
+ await this.uploadAfterCreationOf(folder);
81
+ return fs;
82
+ });
83
+ }
84
+ async syncBeforeChange() {
85
+ const { state } = await this.fs.v.sync.updateStatusInfo('');
86
+ if (state === 'behind') {
87
+ await this.fs.v.sync.adoptRemote('');
88
+ }
89
+ }
90
+ async uploadAfterCreationOf(folder) {
91
+ // XXX must add work with not-online condition
92
+ await this.fs.v.sync.upload(folder);
93
+ await this.fs.v.sync.upload('');
94
+ }
95
+ startSyncProc() {
96
+ this.syncFolderProc = (new rxjs_1.Observable((obs) => this.fs.watchFolder('', obs)))
97
+ .pipe((0, operators_1.filter)(ev => (ev.type === 'remote-change')), (0, operators_1.mergeMap)(() => this.syncP().startOrChain(() => this.fs.v.sync.adoptRemote('')), 1))
98
+ .subscribe();
99
+ }
100
+ stopSync() {
101
+ if (this.syncFolderProc) {
102
+ this.syncFolderProc.unsubscribe();
103
+ this.syncFolderProc = undefined;
104
+ }
105
+ }
106
+ }
107
+ exports.AppDataFolders = AppDataFolders;
108
+ Object.freeze(AppDataFolders.prototype);
109
+ Object.freeze(AppDataFolders);
110
+ Object.freeze(exports);