core-3nweb-client-lib 0.26.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/build/api-defs/asmail.d.ts +1 -1
  2. package/build/api-defs/files.d.ts +278 -69
  3. package/build/core/app-files.js +7 -7
  4. package/build/core/asmail/config/common.js +2 -2
  5. package/build/core/asmail/config/index.js +2 -2
  6. package/build/core/asmail/config/published-intro-key.js +1 -1
  7. package/build/core/asmail/delivery/common.js +7 -7
  8. package/build/core/asmail/delivery/index.js +5 -5
  9. package/build/core/asmail/delivery/msg.js +4 -4
  10. package/build/core/asmail/delivery/per-recipient-wip.js +1 -1
  11. package/build/core/asmail/inbox/attachments/fs.js +5 -1
  12. package/build/core/asmail/inbox/cached-msgs.js +1 -1
  13. package/build/core/asmail/inbox/inbox-events.js +4 -4
  14. package/build/core/asmail/inbox/index.js +10 -10
  15. package/build/core/asmail/inbox/msg-downloader.js +1 -1
  16. package/build/core/asmail/inbox/msg-indexing.js +1 -1
  17. package/build/core/asmail/inbox/msg-on-disk.js +5 -5
  18. package/build/core/asmail/index.d.ts +3 -3
  19. package/build/core/asmail/index.js +13 -8
  20. package/build/core/asmail/key-verification.js +5 -5
  21. package/build/core/asmail/keyring/common.js +7 -6
  22. package/build/core/asmail/keyring/correspondent-keys.js +8 -7
  23. package/build/core/asmail/keyring/id-to-email-map.js +2 -1
  24. package/build/core/asmail/keyring/index.d.ts +7 -8
  25. package/build/core/asmail/keyring/index.js +15 -14
  26. package/build/core/asmail/keyring/keyring-storage.js +2 -1
  27. package/build/core/asmail/msg/opener.js +3 -3
  28. package/build/core/asmail/msg/packer.js +13 -13
  29. package/build/core/asmail/sending-params/own-params.js +2 -2
  30. package/build/core/asmail/sending-params/params-from-others.js +1 -1
  31. package/build/core/id-manager.js +6 -3
  32. package/build/core/index.d.ts +2 -1
  33. package/build/core/index.js +14 -14
  34. package/build/core/sign-in.js +5 -5
  35. package/build/core/sign-up.js +9 -9
  36. package/build/core/storage/common/json-saving.js +2 -2
  37. package/build/core/storage/common/obj-info-file.d.ts +12 -4
  38. package/build/core/storage/common/obj-info-file.js +66 -34
  39. package/build/core/storage/common/utils.d.ts +2 -0
  40. package/build/core/storage/common/utils.js +32 -0
  41. package/build/core/storage/index.d.ts +3 -17
  42. package/build/core/storage/index.js +56 -76
  43. package/build/core/storage/local/obj-files-gc.d.ts +2 -0
  44. package/build/core/storage/local/obj-files-gc.js +49 -37
  45. package/build/core/storage/local/obj-files.d.ts +4 -7
  46. package/build/core/storage/local/obj-files.js +7 -10
  47. package/build/core/storage/local/obj-status.d.ts +12 -6
  48. package/build/core/storage/local/obj-status.js +24 -9
  49. package/build/core/storage/local/storage.d.ts +9 -6
  50. package/build/core/storage/local/storage.js +29 -18
  51. package/build/core/storage/synced/downloader.js +1 -1
  52. package/build/core/storage/synced/obj-files-gc.d.ts +5 -1
  53. package/build/core/storage/synced/obj-files-gc.js +91 -37
  54. package/build/core/storage/synced/obj-files.d.ts +42 -36
  55. package/build/core/storage/synced/obj-files.js +178 -147
  56. package/build/core/storage/synced/obj-status.d.ts +87 -85
  57. package/build/core/storage/synced/obj-status.js +463 -259
  58. package/build/core/storage/synced/remote-events.d.ts +11 -12
  59. package/build/core/storage/synced/remote-events.js +73 -56
  60. package/build/core/storage/synced/storage.d.ts +18 -9
  61. package/build/core/storage/synced/storage.js +108 -48
  62. package/build/core/storage/synced/upload-header-file.d.ts +4 -0
  63. package/build/core/storage/synced/upload-header-file.js +64 -0
  64. package/build/core/storage/synced/upsyncer.d.ts +12 -7
  65. package/build/core/storage/synced/upsyncer.js +205 -280
  66. package/build/core/storage/system-folders/apps-data.d.ts +16 -0
  67. package/build/core/storage/system-folders/apps-data.js +110 -0
  68. package/build/core/storage/system-folders/index.d.ts +18 -0
  69. package/build/core/storage/system-folders/index.js +77 -0
  70. package/build/core-ipc/common-caps.js +3 -3
  71. package/build/core-ipc/generic.js +8 -8
  72. package/build/core-ipc/startup-caps.js +2 -2
  73. package/build/cryptors.js +6 -2
  74. package/build/ipc-via-protobuf/asmail-cap.js +58 -57
  75. package/build/ipc-via-protobuf/bytes.js +16 -17
  76. package/build/ipc-via-protobuf/connector-clients-side.d.ts +1 -0
  77. package/build/ipc-via-protobuf/connector-clients-side.js +14 -15
  78. package/build/ipc-via-protobuf/connector-services-side.js +10 -10
  79. package/build/ipc-via-protobuf/connector.js +4 -4
  80. package/build/ipc-via-protobuf/file.d.ts +48 -12
  81. package/build/ipc-via-protobuf/file.js +474 -126
  82. package/build/ipc-via-protobuf/fs.d.ts +8 -0
  83. package/build/ipc-via-protobuf/fs.js +577 -142
  84. package/build/ipc-via-protobuf/log-cap.js +2 -2
  85. package/build/ipc-via-protobuf/mailerid.js +3 -3
  86. package/build/ipc-via-protobuf/protobuf-msg.d.ts +1 -0
  87. package/build/ipc-via-protobuf/protobuf-msg.js +11 -7
  88. package/build/ipc-via-protobuf/startup-cap.js +21 -21
  89. package/build/ipc-via-protobuf/storage-cap.js +12 -12
  90. package/build/ipc.js +7 -2
  91. package/build/lib-client/3nstorage/exceptions.d.ts +3 -0
  92. package/build/lib-client/3nstorage/exceptions.js +13 -1
  93. package/build/lib-client/3nstorage/service.d.ts +16 -2
  94. package/build/lib-client/3nstorage/service.js +104 -38
  95. package/build/lib-client/3nstorage/util/file-based-json.d.ts +2 -1
  96. package/build/lib-client/3nstorage/util/file-based-json.js +1 -1
  97. package/build/lib-client/3nstorage/xsp-fs/attrs.js +17 -17
  98. package/build/lib-client/3nstorage/xsp-fs/common.d.ts +42 -18
  99. package/build/lib-client/3nstorage/xsp-fs/common.js +29 -19
  100. package/build/lib-client/3nstorage/xsp-fs/file-node.d.ts +1 -0
  101. package/build/lib-client/3nstorage/xsp-fs/file-node.js +17 -13
  102. package/build/lib-client/3nstorage/xsp-fs/file.d.ts +31 -6
  103. package/build/lib-client/3nstorage/xsp-fs/file.js +73 -25
  104. package/build/lib-client/3nstorage/xsp-fs/folder-node-serialization.js +4 -4
  105. package/build/lib-client/3nstorage/xsp-fs/folder-node.d.ts +24 -11
  106. package/build/lib-client/3nstorage/xsp-fs/folder-node.js +575 -179
  107. package/build/lib-client/3nstorage/xsp-fs/fs.d.ts +35 -4
  108. package/build/lib-client/3nstorage/xsp-fs/fs.js +231 -110
  109. package/build/lib-client/3nstorage/xsp-fs/link-node.d.ts +1 -0
  110. package/build/lib-client/3nstorage/xsp-fs/link-node.js +7 -2
  111. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.d.ts +30 -24
  112. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.js +229 -123
  113. package/build/lib-client/3nstorage/xsp-fs/node-persistence.d.ts +1 -1
  114. package/build/lib-client/3nstorage/xsp-fs/node-persistence.js +17 -18
  115. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v1.js +3 -3
  116. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v2.js +53 -53
  117. package/build/lib-client/3nweb-signup.js +4 -4
  118. package/build/lib-client/asmail/recipient.js +15 -15
  119. package/build/lib-client/asmail/sender.js +22 -22
  120. package/build/lib-client/asmail/service-config.js +3 -3
  121. package/build/lib-client/cryptor/cryptor-in-worker.js +18 -16
  122. package/build/lib-client/cryptor/cryptor-wasm.js +1 -1
  123. package/build/lib-client/cryptor/cryptor.js +4 -2
  124. package/build/lib-client/cryptor/cryptor.wasm +0 -0
  125. package/build/lib-client/cryptor/in-proc-js.js +1 -1
  126. package/build/lib-client/cryptor/in-proc-wasm.js +6 -6
  127. package/build/lib-client/cryptor/worker-js.js +2 -2
  128. package/build/lib-client/cryptor/worker-wasm.js +2 -2
  129. package/build/lib-client/files-select.js +1 -1
  130. package/build/lib-client/files.d.ts +1 -1
  131. package/build/lib-client/files.js +71 -6
  132. package/build/lib-client/fs-collection.js +1 -1
  133. package/build/lib-client/fs-sync-utils.d.ts +5 -0
  134. package/build/lib-client/fs-sync-utils.js +61 -0
  135. package/build/lib-client/fs-view.d.ts +14 -0
  136. package/build/lib-client/fs-view.js +33 -0
  137. package/build/lib-client/key-derivation.js +1 -1
  138. package/build/lib-client/local-files/dev-file-sink.js +9 -9
  139. package/build/lib-client/local-files/dev-file-src.js +2 -2
  140. package/build/lib-client/local-files/device-fs.d.ts +1 -1
  141. package/build/lib-client/local-files/device-fs.js +56 -54
  142. package/build/lib-client/logging/log-to-file.d.ts +1 -1
  143. package/build/lib-client/logging/log-to-file.js +7 -7
  144. package/build/lib-client/mailer-id/login.js +7 -7
  145. package/build/lib-client/mailer-id/provisioner.js +12 -12
  146. package/build/lib-client/objs-on-disk/file-writing-proc.js +3 -3
  147. package/build/lib-client/objs-on-disk/obj-folders.js +31 -31
  148. package/build/lib-client/objs-on-disk/obj-on-disk.d.ts +13 -2
  149. package/build/lib-client/objs-on-disk/obj-on-disk.js +24 -9
  150. package/build/lib-client/request-utils.d.ts +1 -0
  151. package/build/lib-client/request-utils.js +13 -13
  152. package/build/lib-client/server-events.d.ts +3 -3
  153. package/build/lib-client/server-events.js +9 -8
  154. package/build/lib-client/service-locator.js +10 -10
  155. package/build/lib-client/user-with-mid-session.js +7 -7
  156. package/build/lib-client/user-with-pkl-session.js +25 -25
  157. package/build/lib-client/ws-utils.js +2 -2
  158. package/build/lib-common/async-cryptor-wrap.js +4 -4
  159. package/build/lib-common/async-fs-node.d.ts +5 -3
  160. package/build/lib-common/async-fs-node.js +16 -16
  161. package/build/lib-common/byte-streaming/pipe.js +1 -1
  162. package/build/lib-common/byte-streaming/wrapping.js +13 -13
  163. package/build/lib-common/canonical-address.js +1 -1
  164. package/build/lib-common/exceptions/error.d.ts +1 -0
  165. package/build/lib-common/exceptions/error.js +7 -6
  166. package/build/lib-common/exceptions/file.js +4 -0
  167. package/build/lib-common/ipc/ws-ipc.js +2 -2
  168. package/build/lib-common/mid-sigs-NaCl-Ed.js +14 -14
  169. package/build/lib-common/objs-on-disk/file-layout.d.ts +19 -0
  170. package/build/lib-common/objs-on-disk/file-layout.js +130 -12
  171. package/build/lib-common/objs-on-disk/obj-file.d.ts +13 -2
  172. package/build/lib-common/objs-on-disk/obj-file.js +96 -35
  173. package/build/lib-common/objs-on-disk/utils.d.ts +1 -0
  174. package/build/lib-common/objs-on-disk/utils.js +3 -3
  175. package/build/lib-common/objs-on-disk/v1-obj-file-format.js +14 -14
  176. package/build/lib-common/processes/labelled-exec-pools.d.ts +1 -1
  177. package/build/lib-common/processes/labelled-exec-pools.js +1 -1
  178. package/build/lib-common/processes/pressure.js +2 -2
  179. package/build/lib-common/processes/synced.js +1 -1
  180. package/build/lib-common/processes/timeout.js +2 -2
  181. package/build/lib-common/random-node.js +7 -7
  182. package/build/lib-common/service-api/3nstorage/owner.d.ts +95 -35
  183. package/build/lib-common/service-api/3nstorage/owner.js +82 -40
  184. package/build/lib-common/service-api/asmail/delivery.js +2 -2
  185. package/build/lib-common/service-api/asmail/retrieval.js +1 -1
  186. package/build/lib-common/timed-cache.d.ts +1 -0
  187. package/build/lib-common/timed-non-weak-cache.d.ts +1 -0
  188. package/build/lib-common/timed-non-weak-cache.js +11 -0
  189. package/build/lib-common/utils-for-observables.js +4 -4
  190. package/build/lib-common/weak-cache.d.ts +1 -0
  191. package/build/lib-common/weak-cache.js +12 -1
  192. package/build/lib-index.d.ts +2 -1
  193. package/build/lib-index.js +10 -7
  194. package/build/protos/asmail.proto.js +12955 -7496
  195. package/build/protos/file.proto.js +4867 -2744
  196. package/build/protos/fs.proto.js +9227 -3768
  197. package/package.json +6 -5
  198. package/protos/file.proto +91 -19
  199. package/protos/fs.proto +107 -8
@@ -1,5 +1,6 @@
1
1
  /// <reference types="node" />
2
2
  import { Layout } from "xsp-files";
3
+ import { DiffInfo } from "../service-api/3nstorage/owner";
3
4
  export interface HeaderChunkInfo {
4
5
  len: number;
5
6
  fileOfs: number;
@@ -60,4 +61,22 @@ export declare class ObjVersionBytesLayout {
60
61
  addSegsOnFile(thisVerOfs: number, len: number, fileOfs: number): void;
61
62
  addBaseSegsOnFile(thisVerOfs: number, baseVerOfs: number, len: number, fileOfs: number): void;
62
63
  getBaseVersion(): number | undefined;
64
+ calcBaseAbsorptionParams(baseLayout: ObjVersionBytesLayout): BaseAbsorptionParams;
65
+ applyAbsorptionParams(params: BaseAbsorptionParams): void;
66
+ diffFromBase(): {
67
+ diff: DiffInfo;
68
+ newSegsPackOrder: FiniteChunk[];
69
+ };
63
70
  }
71
+ interface ChunkCopyOp {
72
+ ofsInSrcFile: number;
73
+ len: number;
74
+ ofsInDstFile: number;
75
+ }
76
+ export interface BaseAbsorptionParams {
77
+ newBytesEnd: number;
78
+ newBase: number | undefined;
79
+ chunkReplacements: Map<number, (NewSegsChunkOnDisk | BaseSegsChunk)[]>;
80
+ copyOps: ChunkCopyOp[];
81
+ }
82
+ export {};
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  /*
3
- Copyright (C) 2019 3NSoft Inc.
3
+ Copyright (C) 2019, 2022 3NSoft Inc.
4
4
 
5
5
  This program is free software: you can redistribute it and/or modify it under
6
6
  the terms of the GNU General Public License as published by the Free Software
@@ -81,7 +81,12 @@ class ObjVersionBytesLayout {
81
81
  return this.segsLayoutFrozen;
82
82
  }
83
83
  isFileComplete() {
84
- return !this.segsChunks.find(chunk => ((chunk.type === 'new') || (chunk.type === 'new-endless')));
84
+ for (const chunk of this.segsChunks) {
85
+ if ((chunk.type === 'new') || (chunk.type === 'new-endless')) {
86
+ return true;
87
+ }
88
+ }
89
+ return true;
85
90
  }
86
91
  getLayoutOfs() {
87
92
  return this.bytesEnd;
@@ -90,8 +95,8 @@ class ObjVersionBytesLayout {
90
95
  return this.headerChunk;
91
96
  }
92
97
  segsLocations(thisVerOfs, len) {
93
- assert_1.assert(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
94
- assert_1.assert(Number.isInteger(len) && (len >= 0));
98
+ (0, assert_1.assert)(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
99
+ (0, assert_1.assert)(Number.isInteger(len) && (len >= 0));
95
100
  const segInfos = [];
96
101
  const end = thisVerOfs + len;
97
102
  for (let i = 0; (i < this.segsChunks.length) && (len > 0); i += 1) {
@@ -118,7 +123,7 @@ class ObjVersionBytesLayout {
118
123
  len -= chop.len;
119
124
  }
120
125
  else {
121
- const chop = json_utils_1.copy(chunk);
126
+ const chop = (0, json_utils_1.copy)(chunk);
122
127
  chop.thisVerOfs = thisVerOfs;
123
128
  if (chunk.type === 'new-endless') {
124
129
  chop.type = 'new';
@@ -178,7 +183,7 @@ class ObjVersionBytesLayout {
178
183
  }
179
184
  addHeader(len, fileOfs) {
180
185
  this.ensureExpectedFileOfs(fileOfs);
181
- assert_1.assert(Number.isInteger(len) && (len >= 0));
186
+ (0, assert_1.assert)(Number.isInteger(len) && (len >= 0));
182
187
  if (this.headerChunk) {
183
188
  throw new Error(`Header added second time to obj version bytes layout`);
184
189
  }
@@ -188,8 +193,8 @@ class ObjVersionBytesLayout {
188
193
  }
189
194
  addSegsOnFile(thisVerOfs, len, fileOfs) {
190
195
  this.ensureExpectedFileOfs(fileOfs);
191
- assert_1.assert(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
192
- assert_1.assert(Number.isInteger(len) && (len >= 0));
196
+ (0, assert_1.assert)(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
197
+ (0, assert_1.assert)(Number.isInteger(len) && (len >= 0));
193
198
  const s = {
194
199
  type: 'new-on-disk', thisVerOfs, len, fileOfs
195
200
  };
@@ -198,9 +203,9 @@ class ObjVersionBytesLayout {
198
203
  }
199
204
  addBaseSegsOnFile(thisVerOfs, baseVerOfs, len, fileOfs) {
200
205
  this.ensureExpectedFileOfs(fileOfs);
201
- assert_1.assert(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
202
- assert_1.assert(Number.isInteger(baseVerOfs) && (baseVerOfs >= 0));
203
- assert_1.assert(Number.isInteger(len) && (len >= 0));
206
+ (0, assert_1.assert)(Number.isInteger(thisVerOfs) && (thisVerOfs >= 0));
207
+ (0, assert_1.assert)(Number.isInteger(baseVerOfs) && (baseVerOfs >= 0));
208
+ (0, assert_1.assert)(Number.isInteger(len) && (len >= 0));
204
209
  const s = {
205
210
  type: 'base-on-disk', thisVerOfs, baseVerOfs, len, fileOfs
206
211
  };
@@ -210,6 +215,119 @@ class ObjVersionBytesLayout {
210
215
  getBaseVersion() {
211
216
  return this.baseVersion;
212
217
  }
218
+ calcBaseAbsorptionParams(baseLayout) {
219
+ (0, assert_1.assert)(this.isFileComplete() && baseLayout.isFileComplete());
220
+ const chunkReplacements = new Map();
221
+ const copyOps = [];
222
+ let newBytesEnd = this.bytesEnd;
223
+ for (let i = 0; i < this.segsChunks.length; i += 1) {
224
+ const chunk = this.segsChunks[i];
225
+ if (chunk.type === 'new-on-disk') {
226
+ continue;
227
+ }
228
+ if (chunk.type !== 'base') {
229
+ throw new Error(`Absorbing base is not implemented on chunk type ${chunk.type}.`);
230
+ }
231
+ const replacements = [];
232
+ // start in base version offset
233
+ let start = chunk.baseVerOfs;
234
+ const end = start + chunk.len;
235
+ for (let j = 0; j < baseLayout.segsChunks.length; j += 1) {
236
+ const bChunk = baseLayout.segsChunks[j];
237
+ if ((bChunk.type !== 'new-on-disk')
238
+ && (bChunk.type !== 'base')) {
239
+ throw new Error(`Absorbing base is not implemented on chunk type ${chunk.type}.`);
240
+ }
241
+ const bEnd = bChunk.thisVerOfs + bChunk.len;
242
+ if (bEnd <= start) {
243
+ continue;
244
+ }
245
+ const cut = Math.min(end, bEnd);
246
+ const len = cut - start;
247
+ const thisVerOfs = chunk.thisVerOfs + (start - chunk.baseVerOfs);
248
+ const ofsIntoBChunk = start - bChunk.thisVerOfs;
249
+ if (bChunk.type === 'new-on-disk') {
250
+ const op = {
251
+ ofsInSrcFile: bChunk.fileOfs + ofsIntoBChunk,
252
+ len,
253
+ ofsInDstFile: newBytesEnd
254
+ };
255
+ copyOps.push(op);
256
+ const replacementChunk = {
257
+ type: 'new-on-disk',
258
+ fileOfs: newBytesEnd,
259
+ len,
260
+ thisVerOfs
261
+ };
262
+ replacements.push(replacementChunk);
263
+ newBytesEnd += len;
264
+ }
265
+ else if (bChunk.type === 'base') {
266
+ const replacementChunk = {
267
+ type: 'base',
268
+ len,
269
+ thisVerOfs,
270
+ baseVerOfs: bChunk.baseVerOfs + ofsIntoBChunk
271
+ };
272
+ replacements.push(replacementChunk);
273
+ }
274
+ else {
275
+ throw new Error(`This shouldn't be unreachable. Unimplemented chunk type?`);
276
+ }
277
+ start = cut;
278
+ if (start === end) {
279
+ break;
280
+ }
281
+ }
282
+ (0, assert_1.assert)(start === end);
283
+ chunkReplacements.set(i, replacements);
284
+ }
285
+ return {
286
+ chunkReplacements, copyOps, newBytesEnd,
287
+ newBase: baseLayout.baseVersion
288
+ };
289
+ }
290
+ applyAbsorptionParams(params) {
291
+ this.bytesEnd = params.newBytesEnd;
292
+ this.baseVersion = params.newBase;
293
+ const changes = [];
294
+ for (const [segInd, newSegs] of params.chunkReplacements.entries()) {
295
+ const initSeg = this.segsChunks[segInd];
296
+ changes.push([initSeg, newSegs]);
297
+ }
298
+ for (const [initSeg, newSegs] of changes) {
299
+ const ind = this.segsChunks.indexOf(initSeg);
300
+ this.segsChunks.splice(ind, 1, ...newSegs);
301
+ }
302
+ }
303
+ diffFromBase() {
304
+ (0, assert_1.assert)(!!this.baseVersion && this.isFileComplete());
305
+ const sections = [];
306
+ const newSegsPackOrder = [];
307
+ let segsSize = 0;
308
+ let ofsInNewSegsPack = 0;
309
+ for (let i = 0; i < this.segsChunks.length; i += 1) {
310
+ const seg = this.segsChunks[i];
311
+ if ((seg.type === 'base') || (seg.type === 'base-on-disk')) {
312
+ sections.push([0, seg.baseVerOfs, seg.len]);
313
+ }
314
+ else {
315
+ // XXX with simplification second element (bizarre 0) will be gone
316
+ sections.push([1, 0, seg.len]);
317
+ newSegsPackOrder.push({
318
+ thisVerOfs: seg.thisVerOfs, len: seg.len,
319
+ });
320
+ ofsInNewSegsPack += seg.len;
321
+ }
322
+ segsSize += seg.len;
323
+ }
324
+ const diff = {
325
+ baseVersion: this.baseVersion,
326
+ sections,
327
+ segsSize
328
+ };
329
+ return { diff, newSegsPackOrder };
330
+ }
213
331
  }
214
332
  exports.ObjVersionBytesLayout = ObjVersionBytesLayout;
215
333
  Object.freeze(ObjVersionBytesLayout.prototype);
@@ -223,7 +341,7 @@ function layoutSectionsToSegsChunks(sections) {
223
341
  }
224
342
  const chunk = layoutSectionToSegsChunk(section);
225
343
  if (chunk.type === 'new-endless') {
226
- assert_1.assert(sections[sections.length - 1] === section, `Infinite layout section must be the last section`);
344
+ (0, assert_1.assert)(sections[sections.length - 1] === section, `Infinite layout section must be the last section`);
227
345
  }
228
346
  else {
229
347
  chunks.push(chunk);
@@ -1,6 +1,7 @@
1
1
  /// <reference types="node" />
2
2
  import { Layout } from 'xsp-files';
3
- import { FiniteSegsChunk } from './file-layout';
3
+ import { FiniteSegsChunk, FiniteChunk } from './file-layout';
4
+ import { DiffInfo } from '../service-api/3nstorage/owner';
4
5
  export declare class ObjVersionFile {
5
6
  private path;
6
7
  private readonly layout;
@@ -9,9 +10,14 @@ export declare class ObjVersionFile {
9
10
  static forExisting(path: string): Promise<ObjVersionFile>;
10
11
  static createNew(path: string): Promise<ObjVersionFile>;
11
12
  private startCreatingFileOnDisk;
12
- moveFile(newPath: string): Promise<void>;
13
+ moveFile(newPath: string, newHeader: Uint8Array | undefined): Promise<void>;
13
14
  removeFile(): Promise<void>;
14
15
  saveLayout(): Promise<void>;
16
+ /**
17
+ * @param fd already openned file descriptor
18
+ * @param ofs is an optional offset at which layout is written, when we need
19
+ * value different from current one in layout.
20
+ */
15
21
  private recordLayout;
16
22
  private withRWFile;
17
23
  getTotalSegsLen(): number;
@@ -29,6 +35,11 @@ export declare class ObjVersionFile {
29
35
  setSegsLayout(layout: Layout, saveLayout: boolean): Promise<void>;
30
36
  truncateEndlessLayout(): void;
31
37
  isFileComplete(): boolean;
38
+ absorbImmediateBaseVersion(baseVer: number, path: string): Promise<void>;
39
+ diffFromBase(): {
40
+ diff: DiffInfo;
41
+ newSegsPackOrder: FiniteChunk[];
42
+ };
32
43
  }
33
44
  export interface ObjFileParsingException extends web3n.RuntimeException {
34
45
  type: 'obj-file-parsing';
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  /*
3
- Copyright (C) 2019 - 2020 3NSoft Inc.
3
+ Copyright (C) 2019 - 2020, 2022 3NSoft Inc.
4
4
 
5
5
  This program is free software: you can redistribute it and/or modify it under
6
6
  the terms of the GNU General Public License as published by the Free Software
@@ -27,6 +27,7 @@ const v1_obj_file_format_1 = require("./v1-obj-file-format");
27
27
  const fs_1 = require("fs");
28
28
  const file_layout_1 = require("./file-layout");
29
29
  const big_endian_1 = require("../big-endian");
30
+ const assert_1 = require("../assert");
30
31
  class ObjVersionFile {
31
32
  constructor(path, layout) {
32
33
  this.path = path;
@@ -35,20 +36,8 @@ class ObjVersionFile {
35
36
  Object.seal(this);
36
37
  }
37
38
  static async forExisting(path) {
38
- const fd = await fs.open(path, 'r');
39
- try {
40
- const layout = await parseObjVersionBytesLayout(fd)
41
- .catch((err) => {
42
- if (err.type === 'obj-file-parsing') {
43
- err.path = path;
44
- }
45
- throw err;
46
- });
47
- return new ObjVersionFile(path, layout);
48
- }
49
- finally {
50
- await fs.close(fd).catch(noop);
51
- }
39
+ const layout = await readLayoutFrom(path);
40
+ return new ObjVersionFile(path, layout);
52
41
  }
53
42
  static async createNew(path) {
54
43
  const layout = file_layout_1.ObjVersionBytesLayout.forNewFile();
@@ -59,10 +48,22 @@ class ObjVersionFile {
59
48
  startCreatingFileOnDisk() {
60
49
  this.writeProc.addStarted(createNewV1File(this.path));
61
50
  }
62
- moveFile(newPath) {
51
+ moveFile(newPath, newHeader) {
63
52
  return this.writeProc.startOrChain(async () => {
53
+ var _a;
64
54
  await fs.rename(this.path, newPath);
65
55
  this.path = newPath;
56
+ if (newHeader) {
57
+ (0, assert_1.assert)(newHeader.length === ((_a = this.layout.headerLocation()) === null || _a === void 0 ? void 0 : _a.len));
58
+ const headerOfs = this.layout.headerLocation().fileOfs;
59
+ const fd = await fs.open(this.path, 'r+');
60
+ try {
61
+ await fs.write(fd, headerOfs, (0, buffer_utils_1.toBuffer)(newHeader));
62
+ }
63
+ finally {
64
+ await fs.close(fd).catch(noop);
65
+ }
66
+ }
66
67
  });
67
68
  }
68
69
  removeFile() {
@@ -71,9 +72,16 @@ class ObjVersionFile {
71
72
  saveLayout() {
72
73
  return this.withRWFile(fd => this.recordLayout(fd));
73
74
  }
74
- async recordLayout(fd) {
75
+ /**
76
+ * @param fd already openned file descriptor
77
+ * @param ofs is an optional offset at which layout is written, when we need
78
+ * value different from current one in layout.
79
+ */
80
+ async recordLayout(fd, ofs) {
75
81
  const layoutBytes = this.layout.toBytes();
76
- const ofs = this.layout.getLayoutOfs();
82
+ if (!ofs) {
83
+ ofs = this.layout.getLayoutOfs();
84
+ }
77
85
  await fs.writeFromBuf(fd, ofs, layoutBytes);
78
86
  await recordLayoutOffsetInV1(fd, ofs);
79
87
  await fs.ftruncate(fd, ofs + layoutBytes.length);
@@ -82,7 +90,7 @@ class ObjVersionFile {
82
90
  return this.writeProc.startOrChain(async () => {
83
91
  const fd = await fs.open(this.path, 'r+')
84
92
  .catch(exc => {
85
- throw error_1.errWithCause(exc, `Can't open for writing obj-version file ${this.path}`);
93
+ throw (0, error_1.errWithCause)(exc, `Can't open for writing obj-version file ${this.path}`);
86
94
  });
87
95
  try {
88
96
  return await action(fd);
@@ -101,7 +109,7 @@ class ObjVersionFile {
101
109
  saveHeader(header, saveLayout) {
102
110
  return this.withRWFile(async (fd) => {
103
111
  const ofs = this.layout.getLayoutOfs();
104
- await fs.writeFromBuf(fd, ofs, buffer_utils_1.toBuffer(header));
112
+ await fs.writeFromBuf(fd, ofs, (0, buffer_utils_1.toBuffer)(header));
105
113
  this.layout.addHeader(header.length, ofs);
106
114
  if (saveLayout) {
107
115
  await this.recordLayout(fd);
@@ -111,7 +119,7 @@ class ObjVersionFile {
111
119
  saveSegs(segsChunks, thisVerOfs, baseVerOfs, saveLayout) {
112
120
  return this.withRWFile(async (fd) => {
113
121
  const ofs = this.layout.getLayoutOfs();
114
- await fs.writeFromBuf(fd, ofs, buffer_utils_1.toBuffer(segsChunks));
122
+ await fs.writeFromBuf(fd, ofs, (0, buffer_utils_1.toBuffer)(segsChunks));
115
123
  if (baseVerOfs === undefined) {
116
124
  this.layout.addSegsOnFile(thisVerOfs, segsChunks.length, ofs);
117
125
  }
@@ -128,18 +136,25 @@ class ObjVersionFile {
128
136
  }
129
137
  async withROFile(action) {
130
138
  const path = this.path;
131
- const fd = await fs.open(path, 'r')
132
- .catch((exc) => {
139
+ let fd;
140
+ try {
141
+ fd = await fs.open(path, 'r');
142
+ }
143
+ catch (exc) {
144
+ // in some use cases version file can be moved, and for this reason
145
+ // we make second attempt if path is newer
133
146
  if (exc.notFound && (path !== this.path)) {
134
- return fs.open(this.path, 'r');
147
+ try {
148
+ fd = await fs.open(this.path, 'r');
149
+ }
150
+ catch (exc) {
151
+ throw (0, error_1.errWithCause)(exc, `Can't open for reading obj version file ${this.path}`);
152
+ }
135
153
  }
136
154
  else {
137
155
  throw exc;
138
156
  }
139
- })
140
- .catch(exc => {
141
- throw error_1.errWithCause(exc, `Can't open for reading obj version file ${this.path}`);
142
- });
157
+ }
143
158
  try {
144
159
  return await action(fd);
145
160
  }
@@ -171,7 +186,7 @@ class ObjVersionFile {
171
186
  return;
172
187
  }
173
188
  await this.withROFile(async (fd) => {
174
- const src = fs_1.createReadStream('', {
189
+ const src = (0, fs_1.createReadStream)('', {
175
190
  fd,
176
191
  autoClose: false,
177
192
  start: chunkInfo.fileOfs,
@@ -204,7 +219,7 @@ class ObjVersionFile {
204
219
  for (const chunk of chunks) {
205
220
  if ((chunk.type === 'new-on-disk')
206
221
  || (chunk.type === 'base-on-disk')) {
207
- const src = fs_1.createReadStream('', {
222
+ const src = (0, fs_1.createReadStream)('', {
208
223
  fd,
209
224
  autoClose: false,
210
225
  start: chunk.fileOfs,
@@ -235,13 +250,43 @@ class ObjVersionFile {
235
250
  isFileComplete() {
236
251
  return this.layout.isFileComplete();
237
252
  }
253
+ async absorbImmediateBaseVersion(baseVer, path) {
254
+ (0, assert_1.assert)(baseVer === this.layout.getBaseVersion());
255
+ const baseLayout = await readLayoutFrom(path);
256
+ const absorptionParams = this.layout.calcBaseAbsorptionParams(baseLayout);
257
+ const src = await fs.open(path, 'r');
258
+ await this.withRWFile(async (fd) => {
259
+ if (absorptionParams.copyOps.length > 0) {
260
+ await this.recordLayout(fd, absorptionParams.newBytesEnd);
261
+ let buf = Buffer.allocUnsafe(absorptionParams.copyOps[0].len);
262
+ for (const op of absorptionParams.copyOps) {
263
+ if (buf.length < op.len) {
264
+ buf = Buffer.allocUnsafe(op.len);
265
+ }
266
+ const chunk = ((op.len < buf.length) ?
267
+ buf.slice(0, op.len) : buf);
268
+ await fs.readToBuf(src, op.ofsInSrcFile, chunk);
269
+ await fs.writeFromBuf(fd, op.ofsInDstFile, chunk);
270
+ }
271
+ }
272
+ this.layout.applyAbsorptionParams(absorptionParams);
273
+ await this.recordLayout(fd);
274
+ })
275
+ .then(() => fs.close(src).catch(noop), async (exc) => {
276
+ await fs.close(src).catch(noop);
277
+ throw exc;
278
+ });
279
+ }
280
+ diffFromBase() {
281
+ return this.layout.diffFromBase();
282
+ }
238
283
  }
239
284
  exports.ObjVersionFile = ObjVersionFile;
240
285
  Object.freeze(ObjVersionFile.prototype);
241
286
  Object.freeze(ObjVersionFile);
242
287
  function noop() { }
243
288
  async function pipeBytes(src, sink) {
244
- const deferred = deferred_1.defer();
289
+ const deferred = (0, deferred_1.defer)();
245
290
  src.pipe(sink, { end: false });
246
291
  src.on('error', (err) => {
247
292
  deferred.reject(err);
@@ -274,8 +319,8 @@ async function parseObjVersionBytesLayout(fd) {
274
319
  throw exc;
275
320
  });
276
321
  const fileStart = fstBytes.slice(0, 4);
277
- if (bytes_equal_1.bytesEqual(fileStart, v1_obj_file_format_1.V1_FILE_START)) {
278
- const layoutOfs = big_endian_1.uintFrom8Bytes(fstBytes, 4);
322
+ if ((0, bytes_equal_1.bytesEqual)(fileStart, v1_obj_file_format_1.V1_FILE_START)) {
323
+ const layoutOfs = (0, big_endian_1.uintFrom8Bytes)(fstBytes, 4);
279
324
  if (layoutOfs === 0) {
280
325
  throw parsingException(`Obj version file is in incomplete state`);
281
326
  }
@@ -302,10 +347,26 @@ async function createNewV1File(path) {
302
347
  }
303
348
  async function recordLayoutOffsetInV1(fd, ofs) {
304
349
  const ofsInBytes = Buffer.allocUnsafe(8);
305
- big_endian_1.packUintTo8Bytes(ofs, ofsInBytes, 0);
350
+ (0, big_endian_1.packUintTo8Bytes)(ofs, ofsInBytes, 0);
306
351
  await fs.writeFromBuf(fd, v1_obj_file_format_1.V1_FILE_START.length, ofsInBytes)
307
352
  .catch(exc => {
308
- throw error_1.errWithCause(exc, `Can't record layout offset in obj file`);
353
+ throw (0, error_1.errWithCause)(exc, `Can't record layout offset in obj file`);
309
354
  });
310
355
  }
356
+ async function readLayoutFrom(path) {
357
+ const fd = await fs.open(path, 'r');
358
+ try {
359
+ const layout = await parseObjVersionBytesLayout(fd)
360
+ .catch((err) => {
361
+ if (err.type === 'obj-file-parsing') {
362
+ err.path = path;
363
+ }
364
+ throw err;
365
+ });
366
+ return layout;
367
+ }
368
+ finally {
369
+ await fs.close(fd).catch(noop);
370
+ }
371
+ }
311
372
  Object.freeze(exports);
@@ -1,4 +1,5 @@
1
1
  /// <reference types="node" />
2
+ /// <reference types="node" />
2
3
  import { Layout } from 'xsp-files';
3
4
  import { DiffInfo } from '../service-api/3nstorage/owner';
4
5
  import { ObjVersionFile } from './obj-file';
@@ -42,7 +42,7 @@ function diffToLayout(diff) {
42
42
  }
43
43
  exports.diffToLayout = diffToLayout;
44
44
  async function streamToObjFile(file, content, src, maxBufferLen) {
45
- let deferred = deferred_1.defer();
45
+ let deferred = (0, deferred_1.defer)();
46
46
  const contentIter = content.values();
47
47
  let chunk = undefined;
48
48
  let buf = new bytes_fifo_buffer_1.BytesFIFOBuffer();
@@ -111,7 +111,7 @@ async function streamToObjFile(file, content, src, maxBufferLen) {
111
111
  });
112
112
  src.on('end', () => {
113
113
  if (chunk) {
114
- complete(file_1.makeFileException(file_1.Code.endOfFile, '<input stream>'));
114
+ complete((0, file_1.makeFileException)(file_1.Code.endOfFile, '<input stream>'));
115
115
  }
116
116
  else {
117
117
  complete();
@@ -125,7 +125,7 @@ exports.streamToObjFile = streamToObjFile;
125
125
  function chunksInOrderedStream(len, headerLen, segsOfs) {
126
126
  const chunks = [];
127
127
  if (typeof headerLen === 'number') {
128
- assert_1.assert(len >= headerLen);
128
+ (0, assert_1.assert)(len >= headerLen);
129
129
  chunks.push({ type: 'header', len: headerLen });
130
130
  len -= headerLen;
131
131
  }
@@ -23,16 +23,16 @@ var headerChunkInfo;
23
23
  (function (headerChunkInfo) {
24
24
  function toBytes(hInfo) {
25
25
  const buf = Buffer.allocUnsafe(12);
26
- big_endian_1.packUintTo4Bytes(hInfo.len, buf, 0);
27
- big_endian_1.packUintTo8Bytes(hInfo.fileOfs, buf, 4);
26
+ (0, big_endian_1.packUintTo4Bytes)(hInfo.len, buf, 0);
27
+ (0, big_endian_1.packUintTo8Bytes)(hInfo.fileOfs, buf, 4);
28
28
  return buf;
29
29
  }
30
30
  headerChunkInfo.toBytes = toBytes;
31
31
  function fromBytes(b, i) {
32
32
  let bytesRead = 0;
33
- const len = big_endian_1.uintFrom4Bytes(b, i + bytesRead);
33
+ const len = (0, big_endian_1.uintFrom4Bytes)(b, i + bytesRead);
34
34
  bytesRead += 4;
35
- const fileOfs = big_endian_1.uintFrom8Bytes(b, i + bytesRead);
35
+ const fileOfs = (0, big_endian_1.uintFrom8Bytes)(b, i + bytesRead);
36
36
  bytesRead += 8;
37
37
  const hInfo = { len, fileOfs };
38
38
  return { hInfo: Object.freeze(hInfo), bytesRead };
@@ -60,18 +60,18 @@ var segsChunkInfo;
60
60
  let i = 0;
61
61
  buf[i] = flag;
62
62
  i += 1;
63
- big_endian_1.packUintTo8Bytes(sInfo.thisVerOfs, buf, i);
63
+ (0, big_endian_1.packUintTo8Bytes)(sInfo.thisVerOfs, buf, i);
64
64
  i += 8;
65
65
  if (sInfo.type !== 'new-endless') {
66
- big_endian_1.packUintTo8Bytes(sInfo.len, buf, 9);
66
+ (0, big_endian_1.packUintTo8Bytes)(sInfo.len, buf, 9);
67
67
  i += 8;
68
68
  }
69
69
  if ((sInfo.type === 'new-on-disk') || (sInfo.type === 'base-on-disk')) {
70
- big_endian_1.packUintTo8Bytes(sInfo.fileOfs, buf, i);
70
+ (0, big_endian_1.packUintTo8Bytes)(sInfo.fileOfs, buf, i);
71
71
  i += 8;
72
72
  }
73
73
  if ((sInfo.type === 'base') || (sInfo.type === 'base-on-disk')) {
74
- big_endian_1.packUintTo8Bytes(sInfo.baseVerOfs, buf, i);
74
+ (0, big_endian_1.packUintTo8Bytes)(sInfo.baseVerOfs, buf, i);
75
75
  }
76
76
  return buf;
77
77
  }
@@ -80,21 +80,21 @@ var segsChunkInfo;
80
80
  let bytesRead = 0;
81
81
  const flag = b[i + bytesRead];
82
82
  bytesRead += 1;
83
- const thisVerOfs = big_endian_1.uintFrom8Bytes(b, i + bytesRead);
83
+ const thisVerOfs = (0, big_endian_1.uintFrom8Bytes)(b, i + bytesRead);
84
84
  bytesRead += 8;
85
85
  let len = undefined;
86
86
  if ((flag & IS_ENDLESS_BITMASK) === 0) {
87
- len = big_endian_1.uintFrom8Bytes(b, i + bytesRead);
87
+ len = (0, big_endian_1.uintFrom8Bytes)(b, i + bytesRead);
88
88
  bytesRead += 8;
89
89
  }
90
90
  let fileOfs = undefined;
91
91
  if (flag & FILE_OFS_PRESENT_BITMASK) {
92
- fileOfs = big_endian_1.uintFrom8Bytes(b, i + bytesRead);
92
+ fileOfs = (0, big_endian_1.uintFrom8Bytes)(b, i + bytesRead);
93
93
  bytesRead += 8;
94
94
  }
95
95
  let baseVerOfs = undefined;
96
96
  if (flag & BASE_VER_OFS_PRESENT_BITMASK) {
97
- baseVerOfs = big_endian_1.uintFrom8Bytes(b, i + bytesRead);
97
+ baseVerOfs = (0, big_endian_1.uintFrom8Bytes)(b, i + bytesRead);
98
98
  bytesRead += 8;
99
99
  }
100
100
  const isOnDisk = (fileOfs !== undefined);
@@ -168,7 +168,7 @@ var layoutV1;
168
168
  if (a.baseVersion !== undefined) {
169
169
  flag |= BASE_PRESENT_BITMASK;
170
170
  baseBytes = Buffer.allocUnsafe(8);
171
- big_endian_1.packUintTo8Bytes(a.baseVersion, baseBytes, 0);
171
+ (0, big_endian_1.packUintTo8Bytes)(a.baseVersion, baseBytes, 0);
172
172
  if (a.allBaseBytesInFile) {
173
173
  flag |= ALL_BASE_BYTES_IN_FILE_BITMASK;
174
174
  }
@@ -214,7 +214,7 @@ var layoutV1;
214
214
  i += 1;
215
215
  let baseVersion = undefined;
216
216
  if (flag & BASE_PRESENT_BITMASK) {
217
- baseVersion = big_endian_1.uintFrom8Bytes(b, i);
217
+ baseVersion = (0, big_endian_1.uintFrom8Bytes)(b, i);
218
218
  i += 8;
219
219
  }
220
220
  let headerChunk = undefined;
@@ -10,7 +10,7 @@ declare class ProcessingPool<PoolLabel extends string> {
10
10
  private readonly inProcess;
11
11
  private readonly queue;
12
12
  isRunning: boolean;
13
- constructor(label: PoolLabel, maxProcs: number, logError?: ((err: any, msg?: string | undefined) => Promise<void>) | undefined);
13
+ constructor(label: PoolLabel, maxProcs: number, logError?: ((err: any, msg?: string) => Promise<void>) | undefined);
14
14
  add(task: Task<any>): void;
15
15
  private processNextQueued;
16
16
  pause(): void;
@@ -92,7 +92,7 @@ class LabelledExecPools {
92
92
  constructor(setup, logError) {
93
93
  this.pools = new Map();
94
94
  this.isRunning = false;
95
- assert_1.assert(setup.length > 0);
95
+ (0, assert_1.assert)(setup.length > 0);
96
96
  for (const { label, maxProcs } of setup) {
97
97
  const pool = new ProcessingPool(label, maxProcs, logError);
98
98
  this.pools.set(pool.label, pool);
@@ -35,7 +35,7 @@ class PressureValve {
35
35
  if (this.stopper) {
36
36
  return;
37
37
  }
38
- this.stopper = deferred_1.defer();
38
+ this.stopper = (0, deferred_1.defer)();
39
39
  }
40
40
  else {
41
41
  if (!this.stopper) {
@@ -47,7 +47,7 @@ class PressureValve {
47
47
  }
48
48
  pressWithError(err) {
49
49
  this.toggle(true);
50
- this.stopper.reject(error_1.errWithCause(err, `Backpressure error`));
50
+ this.stopper.reject((0, error_1.errWithCause)(err, `Backpressure error`));
51
51
  }
52
52
  }
53
53
  exports.PressureValve = PressureValve;
@@ -149,7 +149,7 @@ class SingleProc {
149
149
  }
150
150
  }
151
151
  startOrChain(action) {
152
- const deferred = deferred_1.defer();
152
+ const deferred = (0, deferred_1.defer)();
153
153
  this.actions.push({ action, deferred });
154
154
  this.runIfIdle();
155
155
  return deferred.promise;