core-3nweb-client-lib 0.26.0 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/build/api-defs/asmail.d.ts +1 -1
  2. package/build/api-defs/files.d.ts +278 -69
  3. package/build/core/app-files.js +7 -7
  4. package/build/core/asmail/config/common.js +2 -2
  5. package/build/core/asmail/config/index.js +2 -2
  6. package/build/core/asmail/config/published-intro-key.js +1 -1
  7. package/build/core/asmail/delivery/common.js +7 -7
  8. package/build/core/asmail/delivery/index.js +5 -5
  9. package/build/core/asmail/delivery/msg.js +4 -4
  10. package/build/core/asmail/delivery/per-recipient-wip.js +1 -1
  11. package/build/core/asmail/inbox/attachments/fs.js +5 -1
  12. package/build/core/asmail/inbox/cached-msgs.js +1 -1
  13. package/build/core/asmail/inbox/inbox-events.js +4 -4
  14. package/build/core/asmail/inbox/index.js +10 -10
  15. package/build/core/asmail/inbox/msg-downloader.js +1 -1
  16. package/build/core/asmail/inbox/msg-indexing.js +1 -1
  17. package/build/core/asmail/inbox/msg-on-disk.js +5 -5
  18. package/build/core/asmail/index.d.ts +3 -3
  19. package/build/core/asmail/index.js +13 -8
  20. package/build/core/asmail/key-verification.js +5 -5
  21. package/build/core/asmail/keyring/common.js +7 -6
  22. package/build/core/asmail/keyring/correspondent-keys.js +8 -7
  23. package/build/core/asmail/keyring/id-to-email-map.js +2 -1
  24. package/build/core/asmail/keyring/index.d.ts +7 -8
  25. package/build/core/asmail/keyring/index.js +15 -14
  26. package/build/core/asmail/keyring/keyring-storage.js +2 -1
  27. package/build/core/asmail/msg/opener.js +3 -3
  28. package/build/core/asmail/msg/packer.js +13 -13
  29. package/build/core/asmail/sending-params/own-params.js +2 -2
  30. package/build/core/asmail/sending-params/params-from-others.js +1 -1
  31. package/build/core/id-manager.js +6 -3
  32. package/build/core/index.d.ts +2 -1
  33. package/build/core/index.js +14 -14
  34. package/build/core/sign-in.js +5 -5
  35. package/build/core/sign-up.js +9 -9
  36. package/build/core/storage/common/json-saving.js +2 -2
  37. package/build/core/storage/common/obj-info-file.d.ts +12 -4
  38. package/build/core/storage/common/obj-info-file.js +66 -34
  39. package/build/core/storage/common/utils.d.ts +2 -0
  40. package/build/core/storage/common/utils.js +32 -0
  41. package/build/core/storage/index.d.ts +3 -17
  42. package/build/core/storage/index.js +56 -76
  43. package/build/core/storage/local/obj-files-gc.d.ts +2 -0
  44. package/build/core/storage/local/obj-files-gc.js +49 -37
  45. package/build/core/storage/local/obj-files.d.ts +4 -7
  46. package/build/core/storage/local/obj-files.js +7 -10
  47. package/build/core/storage/local/obj-status.d.ts +12 -6
  48. package/build/core/storage/local/obj-status.js +24 -9
  49. package/build/core/storage/local/storage.d.ts +9 -6
  50. package/build/core/storage/local/storage.js +29 -18
  51. package/build/core/storage/synced/downloader.js +1 -1
  52. package/build/core/storage/synced/obj-files-gc.d.ts +5 -1
  53. package/build/core/storage/synced/obj-files-gc.js +91 -37
  54. package/build/core/storage/synced/obj-files.d.ts +42 -36
  55. package/build/core/storage/synced/obj-files.js +178 -147
  56. package/build/core/storage/synced/obj-status.d.ts +87 -85
  57. package/build/core/storage/synced/obj-status.js +463 -259
  58. package/build/core/storage/synced/remote-events.d.ts +11 -12
  59. package/build/core/storage/synced/remote-events.js +73 -56
  60. package/build/core/storage/synced/storage.d.ts +18 -9
  61. package/build/core/storage/synced/storage.js +108 -48
  62. package/build/core/storage/synced/upload-header-file.d.ts +4 -0
  63. package/build/core/storage/synced/upload-header-file.js +64 -0
  64. package/build/core/storage/synced/upsyncer.d.ts +12 -7
  65. package/build/core/storage/synced/upsyncer.js +204 -280
  66. package/build/core/storage/system-folders/apps-data.d.ts +16 -0
  67. package/build/core/storage/system-folders/apps-data.js +110 -0
  68. package/build/core/storage/system-folders/index.d.ts +18 -0
  69. package/build/core/storage/system-folders/index.js +77 -0
  70. package/build/core-ipc/common-caps.js +3 -3
  71. package/build/core-ipc/generic.js +8 -8
  72. package/build/core-ipc/startup-caps.js +2 -2
  73. package/build/cryptors.js +6 -2
  74. package/build/ipc-via-protobuf/asmail-cap.js +58 -57
  75. package/build/ipc-via-protobuf/bytes.js +16 -17
  76. package/build/ipc-via-protobuf/connector-clients-side.d.ts +3 -0
  77. package/build/ipc-via-protobuf/connector-clients-side.js +61 -24
  78. package/build/ipc-via-protobuf/connector-services-side.js +10 -10
  79. package/build/ipc-via-protobuf/connector.js +4 -4
  80. package/build/ipc-via-protobuf/file.d.ts +48 -12
  81. package/build/ipc-via-protobuf/file.js +474 -126
  82. package/build/ipc-via-protobuf/fs.d.ts +8 -0
  83. package/build/ipc-via-protobuf/fs.js +577 -142
  84. package/build/ipc-via-protobuf/log-cap.js +2 -2
  85. package/build/ipc-via-protobuf/mailerid.js +3 -3
  86. package/build/ipc-via-protobuf/protobuf-msg.d.ts +1 -0
  87. package/build/ipc-via-protobuf/protobuf-msg.js +11 -7
  88. package/build/ipc-via-protobuf/startup-cap.js +21 -21
  89. package/build/ipc-via-protobuf/storage-cap.js +12 -12
  90. package/build/ipc.js +7 -2
  91. package/build/lib-client/3nstorage/exceptions.d.ts +3 -0
  92. package/build/lib-client/3nstorage/exceptions.js +13 -1
  93. package/build/lib-client/3nstorage/service.d.ts +15 -2
  94. package/build/lib-client/3nstorage/service.js +104 -38
  95. package/build/lib-client/3nstorage/util/file-based-json.d.ts +2 -1
  96. package/build/lib-client/3nstorage/util/file-based-json.js +1 -1
  97. package/build/lib-client/3nstorage/xsp-fs/attrs.js +17 -17
  98. package/build/lib-client/3nstorage/xsp-fs/common.d.ts +42 -18
  99. package/build/lib-client/3nstorage/xsp-fs/common.js +29 -19
  100. package/build/lib-client/3nstorage/xsp-fs/file-node.d.ts +1 -0
  101. package/build/lib-client/3nstorage/xsp-fs/file-node.js +17 -13
  102. package/build/lib-client/3nstorage/xsp-fs/file.d.ts +31 -6
  103. package/build/lib-client/3nstorage/xsp-fs/file.js +73 -25
  104. package/build/lib-client/3nstorage/xsp-fs/folder-node-serialization.js +4 -4
  105. package/build/lib-client/3nstorage/xsp-fs/folder-node.d.ts +24 -11
  106. package/build/lib-client/3nstorage/xsp-fs/folder-node.js +575 -179
  107. package/build/lib-client/3nstorage/xsp-fs/fs.d.ts +35 -4
  108. package/build/lib-client/3nstorage/xsp-fs/fs.js +231 -110
  109. package/build/lib-client/3nstorage/xsp-fs/link-node.d.ts +1 -0
  110. package/build/lib-client/3nstorage/xsp-fs/link-node.js +7 -2
  111. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.d.ts +30 -24
  112. package/build/lib-client/3nstorage/xsp-fs/node-in-fs.js +229 -123
  113. package/build/lib-client/3nstorage/xsp-fs/node-persistence.d.ts +1 -1
  114. package/build/lib-client/3nstorage/xsp-fs/node-persistence.js +17 -18
  115. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v1.js +3 -3
  116. package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v2.js +53 -53
  117. package/build/lib-client/3nweb-signup.js +4 -4
  118. package/build/lib-client/asmail/recipient.js +15 -15
  119. package/build/lib-client/asmail/sender.js +22 -22
  120. package/build/lib-client/asmail/service-config.js +3 -3
  121. package/build/lib-client/cryptor/cryptor-in-worker.js +18 -16
  122. package/build/lib-client/cryptor/cryptor-wasm.js +1 -1
  123. package/build/lib-client/cryptor/cryptor.js +4 -2
  124. package/build/lib-client/cryptor/cryptor.wasm +0 -0
  125. package/build/lib-client/cryptor/in-proc-js.js +1 -1
  126. package/build/lib-client/cryptor/in-proc-wasm.js +6 -6
  127. package/build/lib-client/cryptor/worker-js.js +2 -2
  128. package/build/lib-client/cryptor/worker-wasm.js +2 -2
  129. package/build/lib-client/files-select.js +1 -1
  130. package/build/lib-client/files.d.ts +1 -1
  131. package/build/lib-client/files.js +71 -6
  132. package/build/lib-client/fs-collection.js +1 -1
  133. package/build/lib-client/fs-sync-utils.d.ts +5 -0
  134. package/build/lib-client/fs-sync-utils.js +61 -0
  135. package/build/lib-client/fs-view.d.ts +14 -0
  136. package/build/lib-client/fs-view.js +33 -0
  137. package/build/lib-client/key-derivation.js +1 -1
  138. package/build/lib-client/local-files/dev-file-sink.js +9 -9
  139. package/build/lib-client/local-files/dev-file-src.js +2 -2
  140. package/build/lib-client/local-files/device-fs.d.ts +1 -1
  141. package/build/lib-client/local-files/device-fs.js +56 -54
  142. package/build/lib-client/logging/log-to-file.d.ts +1 -1
  143. package/build/lib-client/logging/log-to-file.js +7 -7
  144. package/build/lib-client/mailer-id/login.js +7 -7
  145. package/build/lib-client/mailer-id/provisioner.js +12 -12
  146. package/build/lib-client/objs-on-disk/file-writing-proc.js +3 -3
  147. package/build/lib-client/objs-on-disk/obj-folders.js +31 -31
  148. package/build/lib-client/objs-on-disk/obj-on-disk.d.ts +13 -2
  149. package/build/lib-client/objs-on-disk/obj-on-disk.js +24 -9
  150. package/build/lib-client/request-utils.d.ts +1 -0
  151. package/build/lib-client/request-utils.js +13 -13
  152. package/build/lib-client/server-events.d.ts +3 -3
  153. package/build/lib-client/server-events.js +9 -8
  154. package/build/lib-client/service-locator.js +10 -10
  155. package/build/lib-client/user-with-mid-session.js +7 -7
  156. package/build/lib-client/user-with-pkl-session.js +25 -25
  157. package/build/lib-client/ws-utils.js +2 -2
  158. package/build/lib-common/async-cryptor-wrap.js +4 -4
  159. package/build/lib-common/async-fs-node.d.ts +5 -3
  160. package/build/lib-common/async-fs-node.js +16 -16
  161. package/build/lib-common/byte-streaming/pipe.js +1 -1
  162. package/build/lib-common/byte-streaming/wrapping.js +13 -13
  163. package/build/lib-common/canonical-address.js +1 -1
  164. package/build/lib-common/exceptions/error.d.ts +1 -0
  165. package/build/lib-common/exceptions/error.js +7 -6
  166. package/build/lib-common/exceptions/file.js +4 -0
  167. package/build/lib-common/ipc/ws-ipc.js +2 -2
  168. package/build/lib-common/mid-sigs-NaCl-Ed.js +14 -14
  169. package/build/lib-common/objs-on-disk/file-layout.d.ts +19 -0
  170. package/build/lib-common/objs-on-disk/file-layout.js +130 -12
  171. package/build/lib-common/objs-on-disk/obj-file.d.ts +13 -2
  172. package/build/lib-common/objs-on-disk/obj-file.js +96 -35
  173. package/build/lib-common/objs-on-disk/utils.d.ts +1 -0
  174. package/build/lib-common/objs-on-disk/utils.js +3 -3
  175. package/build/lib-common/objs-on-disk/v1-obj-file-format.js +14 -14
  176. package/build/lib-common/processes/labelled-exec-pools.d.ts +1 -1
  177. package/build/lib-common/processes/labelled-exec-pools.js +1 -1
  178. package/build/lib-common/processes/pressure.js +2 -2
  179. package/build/lib-common/processes/synced.js +1 -1
  180. package/build/lib-common/processes/timeout.js +2 -2
  181. package/build/lib-common/random-node.js +7 -7
  182. package/build/lib-common/service-api/3nstorage/owner.d.ts +95 -39
  183. package/build/lib-common/service-api/3nstorage/owner.js +82 -40
  184. package/build/lib-common/service-api/asmail/delivery.js +2 -2
  185. package/build/lib-common/service-api/asmail/retrieval.js +1 -1
  186. package/build/lib-common/timed-cache.d.ts +1 -0
  187. package/build/lib-common/timed-non-weak-cache.d.ts +1 -0
  188. package/build/lib-common/timed-non-weak-cache.js +11 -0
  189. package/build/lib-common/utils-for-observables.js +4 -4
  190. package/build/lib-common/weak-cache.d.ts +1 -0
  191. package/build/lib-common/weak-cache.js +12 -1
  192. package/build/lib-index.d.ts +2 -1
  193. package/build/lib-index.js +10 -7
  194. package/build/protos/asmail.proto.js +12955 -7496
  195. package/build/protos/file.proto.js +4867 -2744
  196. package/build/protos/fs.proto.js +9227 -3768
  197. package/package.json +6 -5
  198. package/protos/file.proto +91 -19
  199. package/protos/fs.proto +107 -8
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  /*
3
- Copyright (C) 2016 - 2020 3NSoft Inc.
3
+ Copyright (C) 2016 - 2020, 2022 3NSoft Inc.
4
4
 
5
5
  This program is free software: you can redistribute it and/or modify it under
6
6
  the terms of the GNU General Public License as published by the Free Software
@@ -20,6 +20,7 @@ exports.GC = void 0;
20
20
  const fs = require("../../../lib-common/async-fs-node");
21
21
  const synced_1 = require("../../../lib-common/processes/synced");
22
22
  const path_1 = require("path");
23
+ const utils_1 = require("../common/utils");
23
24
  class GC {
24
25
  constructor(rmObjFromCache, rmObjFolder) {
25
26
  this.rmObjFromCache = rmObjFromCache;
@@ -49,54 +50,65 @@ class GC {
49
50
  if (this.wip.size === 0) {
50
51
  [this.wip, this.scheduled] = [this.scheduled, this.wip];
51
52
  }
52
- const obj = getAndRemoveOneFrom(this.wip);
53
+ const obj = (0, utils_1.getAndRemoveOneFrom)(this.wip);
53
54
  if (!obj) {
54
55
  return;
55
56
  }
56
- // calculate versions that should not be removed
57
- const { gcMaxVer, nonGarbage } = obj.getNonGarbageVersions();
58
- // if object is set archived, and there is nothing in it worth keeping,
59
- // whole folder can be removed
60
- if (obj.isArchived()) {
61
- if (nonGarbage.size === 0) {
62
- this.rmObjFromCache(obj);
63
- if (obj.objId) {
64
- await this.rmObjFolder(obj.objId);
65
- }
66
- return;
67
- }
68
- }
69
- // for all other cases, we remove version files that are not worth
70
- // keeping.
71
- const lst = await fs.readdir(obj.objFolder);
72
- const rmProcs = [];
73
- for (const f of lst) {
74
- const ver = parseInt(f);
75
- if (isNaN(ver) || nonGarbage.has(ver)
76
- || (gcMaxVer && (ver >= gcMaxVer))) {
77
- continue;
78
- }
79
- rmProcs.push(fs.unlink(path_1.join(obj.objFolder, f)).catch(noop));
80
- }
81
- if (rmProcs.length > 0) {
82
- await Promise.all(rmProcs);
57
+ try {
58
+ await this.collectIn(obj);
83
59
  }
60
+ catch (err) { }
84
61
  return this.objCollecting();
85
62
  };
86
63
  Object.seal(this);
87
64
  }
65
+ async collectIn(obj) {
66
+ const nonGarbage = obj.statusObj().getNonGarbageVersions();
67
+ if (!(await this.checkAndRemoveWholeObjFolder(obj, nonGarbage))) {
68
+ await removeGarbageFiles(obj, nonGarbage);
69
+ }
70
+ }
71
+ async checkAndRemoveWholeObjFolder(obj, { nonGarbage }) {
72
+ // if object is set archived, and there is nothing in it worth keeping,
73
+ // whole folder can be removed
74
+ if (obj.objId
75
+ && obj.statusObj().isArchived()
76
+ && (nonGarbage.size === 0)) {
77
+ this.rmObjFromCache(obj);
78
+ await this.rmObjFolder(obj.objId);
79
+ return true;
80
+ }
81
+ else {
82
+ return false;
83
+ }
84
+ }
88
85
  }
89
86
  exports.GC = GC;
90
87
  Object.freeze(GC.prototype);
91
88
  Object.freeze(GC);
92
- function getAndRemoveOneFrom(set) {
93
- const iter = set.values();
94
- const { value, done } = iter.next();
95
- if (done) {
96
- return;
89
+ async function removeGarbageFiles(obj, nonGarbage) {
90
+ const lst = await fs.readdir(obj.objFolder);
91
+ const rmProcs = [];
92
+ for (const f of lst) {
93
+ if (canGC(f, nonGarbage)) {
94
+ rmProcs.push(fs.unlink((0, path_1.join)(obj.objFolder, f)).catch(utils_1.noop));
95
+ }
96
+ }
97
+ if (rmProcs.length > 0) {
98
+ await Promise.all(rmProcs);
99
+ }
100
+ }
101
+ function canGC(f, { gcMaxVer, nonGarbage }) {
102
+ const ver = parseInt(f);
103
+ if (isNaN(ver)) {
104
+ return false;
105
+ }
106
+ else if (!nonGarbage.has(ver)
107
+ && (!gcMaxVer || (ver < gcMaxVer))) {
108
+ return true;
109
+ }
110
+ else {
111
+ return false;
97
112
  }
98
- set.delete(value);
99
- return value;
100
113
  }
101
- function noop() { }
102
114
  Object.freeze(exports);
@@ -1,6 +1,7 @@
1
- import { ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
1
+ import { LocalObjStatus, ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
2
2
  import { ObjSource, Subscribe } from 'xsp-files';
3
3
  import { GC } from './obj-files-gc';
4
+ import { ObjStatus } from './obj-status';
4
5
  import { LogError } from '../../../lib-client/logging/log-to-file';
5
6
  export declare class ObjFiles {
6
7
  private readonly folders;
@@ -29,12 +30,8 @@ export declare class LocalObj {
29
30
  getObjSrc(version: number): Promise<ObjSource>;
30
31
  private objSegsGetterFromDisk;
31
32
  saveNewVersion(version: number, encSub: Subscribe): Promise<void>;
32
- isArchived(): boolean;
33
- getCurrentVersionOrThrow(): number;
34
- getNonGarbageVersions(): {
35
- gcMaxVer?: number;
36
- nonGarbage: Set<number>;
37
- };
33
+ localStatus(): LocalObjStatus;
34
+ statusObj(): ObjStatus;
38
35
  removeCurrentVersion(): Promise<void>;
39
36
  removeArchivedVersion(version: number): Promise<void>;
40
37
  }
@@ -28,7 +28,7 @@ class ObjFiles {
28
28
  constructor(folders, logError) {
29
29
  this.folders = folders;
30
30
  this.logError = logError;
31
- this.objs = timed_cache_1.makeTimedCache(60 * 1000);
31
+ this.objs = (0, timed_cache_1.makeTimedCache)(60 * 1000);
32
32
  this.folderAccessSyncProc = new synced_1.NamedProcs();
33
33
  this.gc = new obj_files_gc_1.GC(obj => {
34
34
  if (this.objs.get(obj.objId) === obj) {
@@ -99,7 +99,7 @@ class LocalObj {
99
99
  this.objFolder = objFolder;
100
100
  this.status = status;
101
101
  this.scheduleGC = scheduleGC;
102
- this.verObjs = timed_cache_1.makeTimedCache(60 * 1000);
102
+ this.verObjs = (0, timed_cache_1.makeTimedCache)(60 * 1000);
103
103
  this.objSegsGetterFromDisk = async (ver, ofs, len) => {
104
104
  let obj = this.verObjs.get(ver);
105
105
  if (!obj) {
@@ -120,7 +120,7 @@ class LocalObj {
120
120
  return new LocalObj(objId, objFolder, status, scheduleGC);
121
121
  }
122
122
  path(version) {
123
- return path_1.join(this.objFolder, `${version}.v`);
123
+ return (0, path_1.join)(this.objFolder, `${version}.v`);
124
124
  }
125
125
  async getObjSrc(version) {
126
126
  let obj = this.verObjs.get(version);
@@ -152,14 +152,11 @@ class LocalObj {
152
152
  await this.status.setNewCurrentVersion(version, obj.getBaseVersion());
153
153
  this.scheduleGC(this);
154
154
  }
155
- isArchived() {
156
- return this.status.isArchived();
155
+ localStatus() {
156
+ return this.status;
157
157
  }
158
- getCurrentVersionOrThrow() {
159
- return this.status.getCurrentVersionOrThrow();
160
- }
161
- getNonGarbageVersions() {
162
- return this.status.getNonGarbageVersions();
158
+ statusObj() {
159
+ return this.status;
163
160
  }
164
161
  async removeCurrentVersion() {
165
162
  await this.status.removeCurrentVersion(this.verObjs);
@@ -1,4 +1,4 @@
1
- import { ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
1
+ import { LocalObjStatus, ObjId } from '../../../lib-client/3nstorage/xsp-fs/common';
2
2
  import { VersionsInfo } from '../common/obj-info-file';
3
3
  import { LogError } from '../../../lib-client/logging/log-to-file';
4
4
  export interface ObjStatusInfo {
@@ -6,7 +6,7 @@ export interface ObjStatusInfo {
6
6
  isArchived?: boolean;
7
7
  versions: VersionsInfo;
8
8
  }
9
- export declare class ObjStatus {
9
+ export declare class ObjStatus implements LocalObjStatus {
10
10
  private readonly objFolder;
11
11
  private readonly status;
12
12
  private readonly logError;
@@ -17,13 +17,19 @@ export declare class ObjStatus {
17
17
  private triggerSaveProc;
18
18
  isArchived(): boolean;
19
19
  getCurrentVersionOrThrow(): number;
20
- getNonGarbageVersions(): {
21
- gcMaxVer?: number;
22
- nonGarbage: Set<number>;
23
- };
20
+ getNonGarbageVersions(): NonGarbage;
24
21
  setNewCurrentVersion(newVersion: number, baseVer: number | undefined): Promise<void>;
25
22
  removeCurrentVersion(verObjs: ContainerWithDelete<number>): Promise<void>;
26
23
  removeArchivedVersion(version: number, verObjs: ContainerWithDelete<number>): Promise<void>;
24
+ archiveCurrentVersion(): Promise<void>;
25
+ listVersions(): {
26
+ current?: number;
27
+ archived?: number[];
28
+ };
29
+ }
30
+ export interface NonGarbage {
31
+ gcMaxVer?: number;
32
+ nonGarbage: Set<number>;
27
33
  }
28
34
  interface ContainerWithDelete<T> {
29
35
  delete(key: T): void;
@@ -19,15 +19,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
19
19
  exports.ObjStatus = void 0;
20
20
  const path_1 = require("path");
21
21
  const exceptions_1 = require("../../../lib-client/3nstorage/exceptions");
22
- const json_saving_1 = require("../common/json-saving");
23
22
  const obj_info_file_1 = require("../common/obj-info-file");
23
+ const json_saving_1 = require("../common/json-saving");
24
+ const obj_info_file_2 = require("../common/obj-info-file");
25
+ const assert_1 = require("../../../lib-common/assert");
24
26
  const STATUS_FILE_NAME = 'status';
25
27
  class ObjStatus {
26
28
  constructor(objFolder, status, logError) {
27
29
  this.objFolder = objFolder;
28
30
  this.status = status;
29
31
  this.logError = logError;
30
- this.saveProc = new json_saving_1.JSONSavingProc(path_1.join(this.objFolder, STATUS_FILE_NAME), () => this.status);
32
+ this.saveProc = new json_saving_1.JSONSavingProc((0, path_1.join)(this.objFolder, STATUS_FILE_NAME), () => this.status);
31
33
  Object.freeze(this);
32
34
  }
33
35
  static async readFrom(objFolder, objId, logError) {
@@ -61,17 +63,17 @@ class ObjStatus {
61
63
  }
62
64
  getNonGarbageVersions() {
63
65
  return {
64
- nonGarbage: obj_info_file_1.nonGarbageVersionsIn(this.status.versions),
66
+ nonGarbage: (0, obj_info_file_2.nonGarbageVersionsIn)(this.status.versions),
65
67
  gcMaxVer: this.status.versions.current
66
68
  };
67
69
  }
68
70
  async setNewCurrentVersion(newVersion, baseVer) {
69
- obj_info_file_1.setCurrentVersionIn(this.status.versions, newVersion, baseVer);
71
+ (0, obj_info_file_2.setCurrentVersionIn)(this.status.versions, newVersion, baseVer);
70
72
  await this.triggerSaveProc();
71
73
  }
72
74
  async removeCurrentVersion(verObjs) {
73
75
  this.status.isArchived = true;
74
- const current = obj_info_file_1.rmCurrentVersionIn(this.status.versions);
76
+ const current = (0, obj_info_file_2.rmCurrentVersionIn)(this.status.versions);
75
77
  if (typeof current === 'number') {
76
78
  verObjs.delete(current);
77
79
  }
@@ -79,23 +81,36 @@ class ObjStatus {
79
81
  }
80
82
  async removeArchivedVersion(version, verObjs) {
81
83
  verObjs.delete(version);
82
- obj_info_file_1.rmArchVersionsIn(this.status.versions, version);
84
+ (0, obj_info_file_2.rmArchVersionFrom)(this.status.versions, version);
85
+ await this.triggerSaveProc();
86
+ }
87
+ async archiveCurrentVersion() {
88
+ const versions = this.status.versions;
89
+ (0, assert_1.assert)(!!versions.current);
90
+ (0, obj_info_file_1.addArchived)(versions, versions.current);
83
91
  await this.triggerSaveProc();
84
92
  }
93
+ listVersions() {
94
+ const archived = this.status.versions.archived;
95
+ return {
96
+ current: this.status.versions.current,
97
+ archived: (archived ? archived.slice() : [])
98
+ };
99
+ }
85
100
  }
86
101
  exports.ObjStatus = ObjStatus;
87
102
  Object.freeze(ObjStatus.prototype);
88
103
  Object.freeze(ObjStatus);
89
104
  async function readAndCheckStatus(objFolder, objId) {
90
- const status = await obj_info_file_1.readJSONInfoFileIn(objFolder, STATUS_FILE_NAME);
105
+ const status = await (0, obj_info_file_2.readJSONInfoFileIn)(objFolder, STATUS_FILE_NAME);
91
106
  if (!status) {
92
- throw exceptions_1.makeStorageException({
107
+ throw (0, exceptions_1.makeStorageException)({
93
108
  message: `Obj status file is not found in obj folder ${objFolder}`
94
109
  });
95
110
  }
96
111
  // XXX we may do some checks and sanitization here
97
112
  if (objId !== status.objId) {
98
- throw exceptions_1.makeStorageException({ message: `Invalid objId in status file for obj ${objId}, in folder ${objFolder}.\nInvalid content:\n${JSON.stringify(status, null, 2)}` });
113
+ throw (0, exceptions_1.makeStorageException)({ message: `Invalid objId in status file for obj ${objId}, in folder ${objFolder}.\nInvalid content:\n${JSON.stringify(status, null, 2)}` });
99
114
  }
100
115
  return status;
101
116
  }
@@ -1,26 +1,29 @@
1
- import { Storage, NodesContainer, StorageGetter, ObjId, NodeEvent } from '../../../lib-client/3nstorage/xsp-fs/common';
1
+ import { Storage, NodesContainer, StorageGetter, ObjId, NodeEvent, LocalObjStatus } from '../../../lib-client/3nstorage/xsp-fs/common';
2
2
  import { LogError } from '../../../lib-client/logging/log-to-file';
3
3
  import { AsyncSBoxCryptor, Subscribe, ObjSource } from 'xsp-files';
4
4
  import { Observable } from 'rxjs';
5
5
  declare type FolderEvent = web3n.files.FolderEvent;
6
6
  declare type FileEvent = web3n.files.FileEvent;
7
+ declare type FSType = web3n.files.FSType;
7
8
  export declare class LocalStorage implements Storage {
8
9
  private readonly files;
9
10
  private readonly getStorages;
10
11
  readonly cryptor: AsyncSBoxCryptor;
11
12
  private readonly logError;
12
- readonly type: web3n.files.FSType;
13
+ readonly type = "local";
13
14
  readonly versioned = true;
14
15
  readonly nodes: NodesContainer;
16
+ readonly connect: undefined;
15
17
  private readonly events;
16
18
  private constructor();
17
19
  static makeAndStart(path: string, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, logError: LogError): Promise<Storage>;
18
20
  getNodeEvents(): Observable<NodeEvent>;
19
- broadcastNodeEvent(objId: ObjId, parentObjId: ObjId | undefined, event: FolderEvent | FileEvent): void;
20
- storageForLinking(type: web3n.files.FSType, location?: string): Storage;
21
+ broadcastNodeEvent(objId: ObjId, parentObjId: ObjId | undefined, childObjId: ObjId | undefined, event: FolderEvent | FileEvent): void;
22
+ storageForLinking(type: FSType, location?: string): Storage;
23
+ private getObjOrThrow;
24
+ status(objId: ObjId): Promise<LocalObjStatus>;
21
25
  generateNewObjId(): Promise<string>;
22
- private getObjNonArchOrThrow;
23
- getObj(objId: ObjId): Promise<ObjSource>;
26
+ getObjSrc(objId: ObjId, version?: number): Promise<ObjSource>;
24
27
  saveObj(objId: string, version: number, encSub: Subscribe): Promise<void>;
25
28
  removeObj(objId: string): Promise<void>;
26
29
  close(): Promise<void>;
@@ -13,7 +13,8 @@
13
13
  See the GNU General Public License for more details.
14
14
 
15
15
  You should have received a copy of the GNU General Public License along with
16
- this program. If not, see <http://www.gnu.org/licenses/>. */
16
+ this program. If not, see <http://www.gnu.org/licenses/>.
17
+ */
17
18
  Object.defineProperty(exports, "__esModule", { value: true });
18
19
  exports.LocalStorage = void 0;
19
20
  const common_1 = require("../../../lib-client/3nstorage/xsp-fs/common");
@@ -32,19 +33,20 @@ class LocalStorage {
32
33
  this.type = 'local';
33
34
  this.versioned = true;
34
35
  this.nodes = new common_1.NodesContainer();
36
+ this.connect = undefined;
35
37
  this.events = new utils_for_observables_1.Broadcast();
36
38
  Object.seal(this);
37
39
  }
38
40
  static async makeAndStart(path, getStorages, cryptor, logError) {
39
41
  const files = await obj_files_1.ObjFiles.makeFor(path, logError);
40
42
  const s = new LocalStorage(files, getStorages, cryptor, logError);
41
- return common_1.wrapStorageImplementation(s);
43
+ return (0, common_1.wrapStorageImplementation)(s);
42
44
  }
43
45
  getNodeEvents() {
44
46
  return this.events.event$;
45
47
  }
46
- broadcastNodeEvent(objId, parentObjId, event) {
47
- this.events.next({ objId, parentObjId, event });
48
+ broadcastNodeEvent(objId, parentObjId, childObjId, event) {
49
+ this.events.next({ objId, parentObjId, childObjId, event });
48
50
  }
49
51
  storageForLinking(type, location) {
50
52
  if ((type === 'local') || (type === 'synced')) {
@@ -57,8 +59,22 @@ class LocalStorage {
57
59
  throw new Error(`Getting ${type} storage is not implemented in local storage.`);
58
60
  }
59
61
  }
62
+ async getObjOrThrow(objId, allowArchived = false) {
63
+ const obj = await this.files.findObj(objId);
64
+ if (!obj
65
+ || (!allowArchived && obj.statusObj().isArchived())) {
66
+ throw (0, exceptions_1.makeObjNotFoundExc)(objId);
67
+ }
68
+ else {
69
+ return obj;
70
+ }
71
+ }
72
+ async status(objId) {
73
+ const obj = await this.getObjOrThrow(objId);
74
+ return obj.localStatus();
75
+ }
60
76
  async generateNewObjId() {
61
- const nonce = await random_node_1.bytes(ecma_nacl_1.secret_box.NONCE_LENGTH);
77
+ const nonce = await (0, random_node_1.bytes)(ecma_nacl_1.secret_box.NONCE_LENGTH);
62
78
  const id = buffer_utils_1.base64urlSafe.pack(nonce);
63
79
  if (this.nodes.reserveId(id)) {
64
80
  return id;
@@ -67,35 +83,30 @@ class LocalStorage {
67
83
  return this.generateNewObjId();
68
84
  }
69
85
  }
70
- async getObjNonArchOrThrow(objId) {
71
- const obj = await this.files.findObj(objId);
72
- if (!obj || obj.isArchived()) {
73
- throw exceptions_1.makeObjNotFoundExc(objId);
86
+ async getObjSrc(objId, version) {
87
+ const obj = await this.getObjOrThrow(objId);
88
+ if (!version) {
89
+ version = obj.statusObj().getCurrentVersionOrThrow();
74
90
  }
75
- return obj;
76
- }
77
- async getObj(objId) {
78
- const obj = await this.getObjNonArchOrThrow(objId);
79
- const currentVer = obj.getCurrentVersionOrThrow();
80
- return obj.getObjSrc(currentVer);
91
+ return obj.getObjSrc(version);
81
92
  }
82
93
  async saveObj(objId, version, encSub) {
83
94
  const obj = await this.files.findObj(objId);
84
95
  if (version === 1) {
85
96
  if (obj) {
86
- throw exceptions_1.makeObjExistsExc(objId);
97
+ throw (0, exceptions_1.makeObjExistsExc)(objId);
87
98
  }
88
99
  await this.files.saveFirstVersion(objId, encSub);
89
100
  }
90
101
  else {
91
102
  if (!obj) {
92
- throw exceptions_1.makeObjNotFoundExc(objId);
103
+ throw (0, exceptions_1.makeObjNotFoundExc)(objId);
93
104
  }
94
105
  await obj.saveNewVersion(version, encSub);
95
106
  }
96
107
  }
97
108
  async removeObj(objId) {
98
- const obj = await this.getObjNonArchOrThrow(objId);
109
+ const obj = await this.getObjOrThrow(objId);
99
110
  await obj.removeCurrentVersion();
100
111
  }
101
112
  async close() {
@@ -64,7 +64,7 @@ class Downloader {
64
64
  chunks.push(chunk);
65
65
  ofs += chunk.length;
66
66
  }
67
- return buffer_utils_1.joinByteArrs(chunks);
67
+ return (0, buffer_utils_1.joinByteArrs)(chunks);
68
68
  }
69
69
  });
70
70
  }
@@ -1,8 +1,10 @@
1
1
  import { SynchronizerOnObjId, SyncedObj } from './obj-files';
2
+ import { StorageOwner as RemoteStorage } from '../../../lib-client/3nstorage/service';
2
3
  export declare class GC {
3
4
  private readonly sync;
4
5
  private readonly rmObjFromCache;
5
6
  private readonly rmObjFolder;
7
+ private readonly remote;
6
8
  private isStopped;
7
9
  /**
8
10
  * All gc steps are done in this process.
@@ -18,9 +20,11 @@ export declare class GC {
18
20
  * with wip set.
19
21
  */
20
22
  private scheduled;
21
- constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>);
23
+ constructor(sync: SynchronizerOnObjId, rmObjFromCache: (obj: SyncedObj) => void, rmObjFolder: (objId: string) => Promise<void>, remote: RemoteStorage);
22
24
  scheduleCollection: (obj: SyncedObj) => void;
23
25
  stop(): Promise<void>;
24
26
  private objCollecting;
25
27
  private collectIn;
28
+ private upsyncObjRemovalWhenNeeded;
29
+ private checkAndRemoveWholeObjFolder;
26
30
  }
@@ -17,14 +17,18 @@
17
17
  */
18
18
  Object.defineProperty(exports, "__esModule", { value: true });
19
19
  exports.GC = void 0;
20
+ const obj_files_1 = require("./obj-files");
20
21
  const synced_1 = require("../../../lib-common/processes/synced");
21
22
  const fs = require("../../../lib-common/async-fs-node");
22
23
  const path_1 = require("path");
24
+ const utils_1 = require("../common/utils");
25
+ const upload_header_file_1 = require("./upload-header-file");
23
26
  class GC {
24
- constructor(sync, rmObjFromCache, rmObjFolder) {
27
+ constructor(sync, rmObjFromCache, rmObjFolder, remote) {
25
28
  this.sync = sync;
26
29
  this.rmObjFromCache = rmObjFromCache;
27
30
  this.rmObjFolder = rmObjFolder;
31
+ this.remote = remote;
28
32
  this.isStopped = false;
29
33
  /**
30
34
  * All gc steps are done in this process.
@@ -54,11 +58,11 @@ class GC {
54
58
  if (this.wip.size === 0) {
55
59
  [this.wip, this.scheduled] = [this.scheduled, this.wip];
56
60
  }
57
- const obj = getAndRemoveOneFrom(this.wip);
61
+ const obj = (0, utils_1.getAndRemoveOneFrom)(this.wip);
58
62
  if (!obj) {
59
63
  return;
60
64
  }
61
- await this.sync(obj.objId, () => this.collectIn(obj).catch(noop));
65
+ await this.sync(obj.objId, () => this.collectIn(obj).catch(utils_1.noop));
62
66
  return this.objCollecting();
63
67
  };
64
68
  Object.seal(this);
@@ -67,52 +71,102 @@ class GC {
67
71
  this.isStopped = true;
68
72
  }
69
73
  async collectIn(obj) {
70
- // calculate versions that should not be removed
71
- const { gcMaxVer, nonGarbage } = obj.getNonGarbageVersions();
72
- // if object is set archived, and there is nothing in it worth keeping,
73
- // whole folder can be removed
74
- if ((nonGarbage.size === 0)
75
- && obj.isArchived() && obj.sync().isSyncDone()) {
76
- if (!obj.isStatusFileSaved()) {
77
- return;
78
- }
79
- this.rmObjFromCache(obj);
80
- if (obj.objId) {
81
- await this.rmObjFolder(obj.objId);
82
- }
74
+ if (obj.statusObj().isArchived()) {
75
+ await this.upsyncObjRemovalWhenNeeded(obj);
76
+ }
77
+ const nonGarbage = obj.statusObj().getNonGarbageVersions();
78
+ if (!(await this.checkAndRemoveWholeObjFolder(obj, nonGarbage))) {
79
+ await removeGarbageFiles(obj, nonGarbage);
80
+ }
81
+ }
82
+ async upsyncObjRemovalWhenNeeded(obj) {
83
+ if (!needsUpsyncOfRemoval(obj)) {
83
84
  return;
84
85
  }
85
- // for all other cases, we remove version files that are not worth
86
- // keeping.
87
- const lst = await fs.readdir(obj.objFolder).catch(noop);
88
- if (!lst) {
86
+ try {
87
+ await this.remote.deleteObj(obj.objId);
88
+ }
89
+ catch (exc) {
90
+ // XXX will need to schedule other attempt to delete on remote
89
91
  return;
90
92
  }
91
- const rmProcs = [];
92
- for (const f of lst) {
93
- const ver = parseInt(f);
94
- if (isNaN(ver) || nonGarbage.has(ver)
95
- || (gcMaxVer && (ver >= gcMaxVer))) {
96
- continue;
97
- }
98
- rmProcs.push(fs.unlink(path_1.join(obj.objFolder, f)).catch(noop));
93
+ await obj.statusObj().recordSyncOfObjRemoval();
94
+ }
95
+ async checkAndRemoveWholeObjFolder(obj, { local, remote }) {
96
+ if (obj.objId
97
+ && obj.statusObj().isArchived()
98
+ && !needsUpsyncOfRemoval(obj)
99
+ && (!local || (local.nonGarbage.size === 0))
100
+ && (remote.nonGarbage.size === 0)) {
101
+ this.rmObjFromCache(obj);
102
+ await this.rmObjFolder(obj.objId);
103
+ return true;
99
104
  }
100
- if (rmProcs.length > 0) {
101
- await Promise.all(rmProcs);
105
+ else {
106
+ return false;
102
107
  }
103
108
  }
104
109
  }
105
110
  exports.GC = GC;
106
111
  Object.freeze(GC.prototype);
107
112
  Object.freeze(GC);
108
- function getAndRemoveOneFrom(set) {
109
- const iter = set.values();
110
- const { value, done } = iter.next();
111
- if (done) {
113
+ function needsUpsyncOfRemoval(obj) {
114
+ const s = obj.statusObj();
115
+ return !(s.neverUploaded() || (s.syncStatus().state === 'synced'));
116
+ }
117
+ function canRemove(f, { local, remote, uploadVersion }) {
118
+ if (f.endsWith(obj_files_1.UNSYNCED_FILE_NAME_EXT)) {
119
+ const verStr = f.slice(0, f.length - 1 - obj_files_1.UNSYNCED_FILE_NAME_EXT.length);
120
+ return (local ? canGC(verStr, local) : true);
121
+ }
122
+ else if (f.endsWith(obj_files_1.REMOTE_FILE_NAME_EXT)) {
123
+ const verStr = f.slice(0, f.length - 1 - obj_files_1.REMOTE_FILE_NAME_EXT.length);
124
+ return canGC(verStr, remote);
125
+ }
126
+ else if (!!uploadVersion
127
+ && f.endsWith(upload_header_file_1.UPLOAD_HEADER_FILE_NAME_EXT)) {
128
+ const verStr = f.slice(0, f.length - 1 - upload_header_file_1.UPLOAD_HEADER_FILE_NAME_EXT.length);
129
+ return canGCUploadHeader(verStr, uploadVersion);
130
+ }
131
+ else {
132
+ return false;
133
+ }
134
+ }
135
+ function canGC(verStr, nonGC) {
136
+ const ver = parseInt(verStr);
137
+ if (isNaN(ver)) {
138
+ return true;
139
+ }
140
+ else if (!nonGC.nonGarbage.has(ver)
141
+ && (!nonGC.gcMaxVer || (ver < nonGC.gcMaxVer))) {
142
+ return true;
143
+ }
144
+ else {
145
+ return false;
146
+ }
147
+ }
148
+ function canGCUploadHeader(verStr, uploadVersion) {
149
+ const ver = parseInt(verStr);
150
+ if (isNaN(ver)) {
151
+ return true;
152
+ }
153
+ else {
154
+ return (ver === uploadVersion);
155
+ }
156
+ }
157
+ async function removeGarbageFiles(obj, nonGarbage) {
158
+ const lst = await fs.readdir(obj.objFolder).catch(utils_1.noop);
159
+ if (!lst) {
112
160
  return;
113
161
  }
114
- set.delete(value);
115
- return value;
162
+ const rmProcs = [];
163
+ for (const f of lst) {
164
+ if (canRemove(f, nonGarbage)) {
165
+ rmProcs.push(fs.unlink((0, path_1.join)(obj.objFolder, f)).catch(utils_1.noop));
166
+ }
167
+ }
168
+ if (rmProcs.length > 0) {
169
+ await Promise.all(rmProcs);
170
+ }
116
171
  }
117
- function noop() { }
118
172
  Object.freeze(exports);