core-3nweb-client-lib 0.49.6 → 0.49.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/build/api-defs/files.d.ts +11 -5
  2. package/build/core/asmail/delivery/common.d.ts +8 -28
  3. package/build/core/asmail/delivery/common.js +1 -47
  4. package/build/core/asmail/delivery/index.js +3 -3
  5. package/build/core/asmail/delivery/per-recipient-wip.js +3 -2
  6. package/build/core/asmail/inbox/index.d.ts +1 -2
  7. package/build/core/asmail/inbox/index.js +14 -15
  8. package/build/core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.d.ts +11 -0
  9. package/build/core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.js +25 -0
  10. package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.d.ts +33 -0
  11. package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.js +72 -0
  12. package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.d.ts +28 -0
  13. package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.js +477 -0
  14. package/build/core/asmail/inbox/msg-indexing/index.d.ts +12 -26
  15. package/build/core/asmail/inbox/msg-indexing/index.js +47 -68
  16. package/build/core/asmail/inbox/msg-on-disk.js +1 -2
  17. package/build/core/asmail/index.js +8 -7
  18. package/build/core/asmail/msg/common.d.ts +0 -1
  19. package/build/core/asmail/msg/common.js +1 -1
  20. package/build/core/index.js +7 -4
  21. package/build/core/keyring/common.d.ts +1 -0
  22. package/build/core/keyring/common.js +2 -1
  23. package/build/core/keyring/index.d.ts +7 -44
  24. package/build/core/keyring/index.js +89 -222
  25. package/build/core/keyring/introductory-keys/published-intro-key.d.ts +17 -0
  26. package/build/core/keyring/introductory-keys/published-intro-key.js +190 -0
  27. package/build/core/keyring/peer-keys/dataset/v1/json-files.d.ts +105 -0
  28. package/build/core/keyring/peer-keys/dataset/v1/json-files.js +89 -0
  29. package/build/core/keyring/peer-keys/dataset/v2/logs.d.ts +43 -0
  30. package/build/core/keyring/peer-keys/dataset/v2/logs.js +116 -0
  31. package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.d.ts +73 -0
  32. package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.js +543 -0
  33. package/build/core/keyring/peer-keys/index.d.ts +24 -0
  34. package/build/core/keyring/peer-keys/index.js +208 -0
  35. package/build/core/storage/synced/obj-files.js +10 -3
  36. package/build/core/storage/synced/obj-status.d.ts +1 -1
  37. package/build/core/storage/synced/obj-status.js +8 -2
  38. package/build/core/storage/synced/storage.d.ts +4 -4
  39. package/build/core/storage/system-folders/apps-data.d.ts +0 -1
  40. package/build/core/storage/system-folders/apps-data.js +2 -6
  41. package/build/lib-client/fs-utils/fs-sync-utils.d.ts +1 -1
  42. package/build/lib-client/fs-utils/fs-sync-utils.js +70 -13
  43. package/build/lib-client/xsp-fs/fs.js +3 -1
  44. package/build/lib-client/xsp-fs/node-in-fs.d.ts +1 -0
  45. package/build/lib-client/xsp-fs/node-in-fs.js +7 -2
  46. package/build/lib-client/xsp-fs/node-persistence.js +1 -3
  47. package/build/lib-client/xsp-fs/xsp-payload-v2.js +1 -2
  48. package/build/lib-common/dataset-sync/per-device-json-array-logs.d.ts +58 -0
  49. package/build/lib-common/dataset-sync/per-device-json-array-logs.js +432 -0
  50. package/build/lib-common/number-line.d.ts +10 -0
  51. package/build/lib-common/number-line.js +66 -0
  52. package/build/lib-common/objs-on-disk/obj-version-file.d.ts +1 -1
  53. package/build/lib-common/objs-on-disk/obj-version-file.js +3 -3
  54. package/build/lib-common/rng-def.d.ts +1 -0
  55. package/build/lib-common/rng-def.js +8 -0
  56. package/build/lib-sqlite-on-3nstorage/for-sqlite.d.ts +38 -0
  57. package/build/lib-sqlite-on-3nstorage/for-sqlite.js +162 -0
  58. package/build/lib-sqlite-on-3nstorage/index.d.ts +18 -5
  59. package/build/lib-sqlite-on-3nstorage/index.js +49 -26
  60. package/build/tests/caps-api/file-sink-checks/splice.js +72 -1
  61. package/build/tests/caps-api/fs-checks/sync-on-one-dev/upload.js +27 -1
  62. package/build/tests/caps-api/fs-checks/sync-with-two-devs/update-propagation.js +12 -3
  63. package/build/tests/computer.3nweb.core/inbox/msg-indexing.js +16 -12
  64. package/build/tests/computer.3nweb.core/keyrings.js +113 -28
  65. package/build/tests/jasmine.js +2 -2
  66. package/build/tests/units/number-line.js +9 -9
  67. package/package.json +2 -1
  68. package/build/core/asmail/inbox/msg-indexing/logs-n-entries.d.ts +0 -52
  69. package/build/core/asmail/inbox/msg-indexing/logs-n-entries.js +0 -333
  70. package/build/core/asmail/inbox/msg-indexing/sql-indexing.d.ts +0 -4
  71. package/build/core/asmail/inbox/msg-indexing/sql-indexing.js +0 -319
  72. package/build/core/keyring/correspondent-keys.d.ts +0 -149
  73. package/build/core/keyring/correspondent-keys.js +0 -403
  74. package/build/core/keyring/id-to-email-map.d.ts +0 -38
  75. package/build/core/keyring/id-to-email-map.js +0 -105
  76. package/build/core/keyring/keyring-storage.d.ts +0 -9
  77. package/build/core/keyring/keyring-storage.js +0 -54
  78. package/build/core/keyring/published-intro-key.d.ts +0 -45
  79. package/build/core/keyring/published-intro-key.js +0 -200
@@ -0,0 +1,208 @@
1
+ "use strict";
2
+ /*
3
+ Copyright (C) 2015 - 2018, 2022, 2026 3NSoft Inc.
4
+
5
+ This program is free software: you can redistribute it and/or modify it under
6
+ the terms of the GNU General Public License as published by the Free Software
7
+ Foundation, either version 3 of the License, or (at your option) any later
8
+ version.
9
+
10
+ This program is distributed in the hope that it will be useful, but
11
+ WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13
+ See the GNU General Public License for more details.
14
+
15
+ You should have received a copy of the GNU General Public License along with
16
+ this program. If not, see <http://www.gnu.org/licenses/>.
17
+ */
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ exports.makePeersKeyring = makePeersKeyring;
20
+ const common_1 = require("../common");
21
+ const ecma_nacl_1 = require("ecma-nacl");
22
+ const buffer_utils_1 = require("../../../lib-common/buffer-utils");
23
+ const fs_sync_utils_1 = require("../../../lib-client/fs-utils/fs-sync-utils");
24
+ const per_device_json_array_logs_1 = require("../../../lib-common/dataset-sync/per-device-json-array-logs");
25
+ const synced_1 = require("../../../lib-common/processes/synced");
26
+ const logs_1 = require("./dataset/v2/logs");
27
+ const peer_keys_db_1 = require("./dataset/v2/peer-keys-db");
28
+ const canonical_address_1 = require("../../../lib-common/canonical-address");
29
+ const LOGS_DIR = 'peer-keys-logs';
30
+ const PEER_KEYS_FOLDER = 'peer-keys-db';
31
+ const NUM_OF_LOGS_TO_START_CUT = 200;
32
+ async function makePeersKeyring(keyringFS, localFS, random, logger) {
33
+ const logsFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(keyringFS, LOGS_DIR);
34
+ const logsLocalFS = await localFS.writableSubRoot(LOGS_DIR);
35
+ const peerKeysFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(keyringFS, PEER_KEYS_FOLDER);
36
+ // XXX log and saving reduction(!):
37
+ // - we don't need to log message counts
38
+ // - we don't have to immediately save changes in message counts
39
+ // - we want to save and log crypto material, more so, we generate and log new crypto first, then send message
40
+ // - logging and saving of marking pair as in use also doesn't have to be immediate (should it be logged?)
41
+ const logs = await (0, logs_1.makePeerKeysChangeLogs)(logsFS, logsLocalFS, logger.logError);
42
+ const keysDB = await (0, peer_keys_db_1.makePeerKeysDB)(peerKeysFS, random);
43
+ async function suggestPairForPeerIntroKey(peerAddr, peerIntroPKey) {
44
+ const peerCAddr = (0, canonical_address_1.toCanonicalAddress)(peerAddr);
45
+ const alreadySuggested = keysDB.getPairSuggestedToPeer(peerCAddr);
46
+ if (alreadySuggested && (alreadySuggested.senderKid === peerIntroPKey.kid)) {
47
+ return alreadySuggested;
48
+ }
49
+ else {
50
+ if (!keysDB.getPeerAddressForCanonical(peerCAddr)) {
51
+ keysDB.addPeer(peerAddr);
52
+ }
53
+ const { nextCrypto, entryToLog } = await keysDB.generateSuggestedPairOnPeerIntroKey(peerCAddr, peerIntroPKey);
54
+ await keysDB.saveIfNeeded();
55
+ await logs.recordSuggestingCryptoToPeer(entryToLog, peerAddr);
56
+ return nextCrypto;
57
+ }
58
+ }
59
+ function needIntroKeyFor(peerAddr) {
60
+ return !keysDB.getSendingPair((0, canonical_address_1.toCanonicalAddress)(peerAddr));
61
+ }
62
+ async function getSendingCryptoWithinEstablishedPair(peerCAddr) {
63
+ const pair = keysDB.getSendingPair(peerCAddr);
64
+ if (!pair) {
65
+ throw new Error(`No sending pair for peer ${peerCAddr}`);
66
+ }
67
+ const { pids, peerKId, pairAlg, senderKId, senderPKey, msgMasterKey, sentMsgsCount } = pair;
68
+ const msgCount = sentMsgsCount + 1;
69
+ keysDB.updateSentMsgCountInSendingPair(peerCAddr, msgCount);
70
+ let nextMsgCrypto = keysDB.getPairSuggestedToPeer(peerCAddr);
71
+ if (!nextMsgCrypto) {
72
+ const generated = await keysDB.generateRegularSuggestedPairIfNeeded(peerCAddr);
73
+ if (generated) {
74
+ nextMsgCrypto = generated.nextCrypto;
75
+ await keysDB.saveIfNeeded();
76
+ await logs.recordSuggestingCryptoToPeer(generated.entryToLog, peerCAddr);
77
+ }
78
+ }
79
+ return {
80
+ currentPair: {
81
+ pid: selectPid(pids),
82
+ recipientKid: peerKId,
83
+ senderPKey: {
84
+ alg: pairAlg,
85
+ kid: senderKId,
86
+ k: senderPKey,
87
+ use: common_1.KEY_USE.PUBLIC
88
+ }
89
+ },
90
+ msgMasterKey,
91
+ msgCount,
92
+ nextMsgCrypto
93
+ };
94
+ }
95
+ async function markPairAsInUse(peerCAddr, peerKId, recipientKId) {
96
+ const pairFoundAndMarked = keysDB.markPairAsInUse(peerCAddr, peerKId, recipientKId);
97
+ if (pairFoundAndMarked) {
98
+ await keysDB.saveIfNeeded();
99
+ await logs.recordPeerStartedUsingPair(peerCAddr, peerKId, recipientKId);
100
+ }
101
+ }
102
+ async function absorbNextPairSuggestedByPeer(peerAddr, pair, introKey) {
103
+ const peerCAddr = (0, canonical_address_1.toCanonicalAddress)(peerAddr);
104
+ const msgMasterKeyAlg = ecma_nacl_1.box.JWK_ALG_NAME;
105
+ const peerPKey = buffer_utils_1.base64.open(pair.recipientPKey.k);
106
+ if (keysDB.getPeerAddressForCanonical(peerCAddr)) {
107
+ const prevPair = keysDB.getSendingPair(peerCAddr);
108
+ if (introKey) {
109
+ if (prevPair
110
+ && ((introKey.pkey.kid === pair.senderKid) && (prevPair.peerKId === pair.recipientPKey.kid))) {
111
+ // ignoring duplicate
112
+ return;
113
+ }
114
+ const senderSKey = buffer_utils_1.base64.open(introKey.skey.k);
115
+ const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
116
+ keysDB.updateSendingPairFromSuggested(peerCAddr, pair, {
117
+ msgMasterKey, msgMasterKeyAlg, peerPKey, senderSKey,
118
+ senderKId: introKey.pkey.kid,
119
+ senderPKey: introKey.pkey.k
120
+ });
121
+ }
122
+ else {
123
+ if (prevPair
124
+ && ((prevPair.senderKId === pair.senderKid) && (prevPair.peerKId === pair.recipientPKey.kid))) {
125
+ // ignoring duplicate
126
+ return;
127
+ }
128
+ const foundPair = keysDB.findKeyToMatchSuggestedForSendingPair(peerCAddr, pair.senderKid);
129
+ if (!foundPair) {
130
+ // ignore bad
131
+ return;
132
+ }
133
+ const { pkey: senderPKey, skey: senderSKey, pairAlg } = foundPair;
134
+ const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
135
+ if (prevPair) {
136
+ keysDB.updateSendingPairFromSuggested(peerCAddr, pair, {
137
+ msgMasterKey, msgMasterKeyAlg, peerPKey, senderSKey, senderPKey,
138
+ senderKId: pair.senderKid
139
+ });
140
+ }
141
+ else {
142
+ keysDB.addSendingPairFromSuggested(peerCAddr, pair, {
143
+ msgMasterKey, msgMasterKeyAlg, senderSKey, peerPKey, pairAlg, senderPKey,
144
+ senderKId: pair.senderKid
145
+ });
146
+ }
147
+ }
148
+ }
149
+ else {
150
+ if (!introKey) {
151
+ throw new Error(`Peer ${peerAddr} is not known, while used introductory key is not given`);
152
+ }
153
+ keysDB.addPeer(peerAddr);
154
+ const senderSKey = buffer_utils_1.base64.open(introKey.skey.k);
155
+ const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
156
+ keysDB.addSendingPairFromSuggested(peerCAddr, pair, {
157
+ msgMasterKey, msgMasterKeyAlg, senderSKey, peerPKey,
158
+ pairAlg: introKey.pkey.alg,
159
+ senderKId: introKey.pkey.kid,
160
+ senderPKey: introKey.pkey.k
161
+ });
162
+ }
163
+ const entryToLog = keysDB.getSendingPair(peerCAddr);
164
+ await keysDB.saveIfNeeded();
165
+ await logs.recordGettingCryptoFromPeer(entryToLog, peerAddr);
166
+ }
167
+ async function periodicCheckAndUploadOfData() {
168
+ // XXX disable uploading for a moment by commenting out
169
+ // if (logs.numOfLogs() >= NUM_OF_LOGS_TO_START_CUT) {
170
+ // const syncedIndexDbVersion = await keysDB.saveAndSync();
171
+ // await logs.cutLogOnDatasetSyncAndUpload(syncedIndexDbVersion);
172
+ // }
173
+ }
174
+ function millisBeforeNextRun() {
175
+ return 30 * 60 * 1000 + Math.floor(10 * 60 * 1000 * Math.random());
176
+ }
177
+ const { stopSyncing, changeProc } = (0, per_device_json_array_logs_1.startDatasetSync)(periodicCheckAndUploadOfData, millisBeforeNextRun, {
178
+ watchAndApplyOpsFromOtherDevices: logs.watchAndApplyOpsFromOtherDevices
179
+ }, {
180
+ absorbOpsFromOtherDevices: keysDB.absorbOpsFromOtherDevices,
181
+ watchAndApplyOpsFromOtherDevices: keysDB.watchAndApplyOpsFromOtherDevices
182
+ });
183
+ // XXX
184
+ // - check if indexed latest is too old, chopping to the latest shard
185
+ async function close() {
186
+ stopSyncing();
187
+ }
188
+ return {
189
+ getPeerAddressForCanonical: keysDB.getPeerAddressForCanonical,
190
+ needIntroKeyFor,
191
+ suggestPairForPeerIntroKey: (0, synced_1.makeSyncedFunc)(changeProc, undefined, suggestPairForPeerIntroKey),
192
+ getSendingCryptoWithinEstablishedPair: (0, synced_1.makeSyncedFunc)(changeProc, undefined, getSendingCryptoWithinEstablishedPair),
193
+ absorbNextPairSuggestedByPeer: (0, synced_1.makeSyncedFunc)(changeProc, undefined, absorbNextPairSuggestedByPeer),
194
+ findEstablishedReceptionPairs: keysDB.findReceptionPairsForPId,
195
+ markPairAsInUse: (0, synced_1.makeSyncedFunc)(changeProc, undefined, markPairAsInUse),
196
+ updateReceivedMsgCountIn: keysDB.updateReceivedMsgCountIn,
197
+ close,
198
+ getPeerKeysInfo: keysDB.getPeerKeysInfo
199
+ };
200
+ }
201
+ function selectPid(pids) {
202
+ if (pids.length < 1) {
203
+ throw new Error("There are no pair ids in array.");
204
+ }
205
+ const i = Math.round((pids.length - 1) * Math.random());
206
+ return pids[i];
207
+ }
208
+ Object.freeze(exports);
@@ -231,7 +231,14 @@ class SyncedObj {
231
231
  const status = await obj_status_1.ObjStatus.makeForDownloadedVersion(objFolder, objId, version, version, logError);
232
232
  const obj = new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
233
233
  const fPath = obj.remoteVerPath(version);
234
- const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk, parts);
234
+ const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk, parts).catch((exc) => {
235
+ if (exc.alreadyExists) {
236
+ return obj_on_disk_1.ObjOnDisk.forExistingFile(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk);
237
+ }
238
+ else {
239
+ throw exc;
240
+ }
241
+ });
235
242
  obj.remoteVers.set(version, objVer);
236
243
  return obj;
237
244
  }
@@ -397,7 +404,7 @@ class SyncedObj {
397
404
  this.scheduleSelfGC();
398
405
  }
399
406
  async isRemoteVersionOnDisk(version) {
400
- if (!this.status.isAmongRemote(version)) {
407
+ if (!this.status.isAmongRemoteOrSynced(version)) {
401
408
  throw (0, exceptions_1.makeObjVersionNotFoundExc)(this.objId, version);
402
409
  }
403
410
  const verPath = this.remoteVerPath(version);
@@ -408,7 +415,7 @@ class SyncedObj {
408
415
  return (objVer.doesFileNeedDownload() ? 'partial' : 'complete');
409
416
  }
410
417
  async getNumOfBytesNeedingDownload(version) {
411
- if (!this.status.isAmongRemote(version)) {
418
+ if (!this.status.isAmongRemoteOrSynced(version)) {
412
419
  if (this.status.isLocalOnlyVersion(version)) {
413
420
  return 0;
414
421
  }
@@ -147,7 +147,7 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
147
147
  neverUploaded(): boolean;
148
148
  versionBeforeUnsyncedRemoval(): number | undefined;
149
149
  adoptRemoteVersion(version?: number): Promise<void>;
150
- isAmongRemote(version: number): boolean;
150
+ isAmongRemoteOrSynced(version: number): boolean;
151
151
  isLocalOnlyVersion(version: number): boolean;
152
152
  }
153
153
  export declare function readAndCheckStatus(objFolder: string, objId: ObjId): Promise<ObjStatusInfo>;
@@ -553,14 +553,20 @@ class ObjStatus {
553
553
  this.updateStateIndicator();
554
554
  await this.triggerSaveProc();
555
555
  }
556
- isAmongRemote(version) {
557
- var _a;
556
+ isAmongRemoteOrSynced(version) {
557
+ var _a, _b;
558
558
  if (this.status.remote.current === version) {
559
559
  return true;
560
560
  }
561
561
  if ((_a = this.status.remote.archived) === null || _a === void 0 ? void 0 : _a.includes(version)) {
562
562
  return true;
563
563
  }
564
+ if (this.status.remote.baseToDiff[version]) {
565
+ return true;
566
+ }
567
+ if (((_b = this.status.synced) === null || _b === void 0 ? void 0 : _b.version) === version) {
568
+ return true;
569
+ }
564
570
  return false;
565
571
  }
566
572
  isLocalOnlyVersion(version) {
@@ -1,5 +1,5 @@
1
1
  import { IGetMailerIdSigner } from '../../../lib-client/user-with-mid-session';
2
- import { SyncedStorage as ISyncedStorage, NodesContainer, Storage as IStorage, StorageGetter, ObjId, NodeEvent, SyncedObjStatus, DownloadEventSink } from '../../../lib-client/xsp-fs/common';
2
+ import { SyncedStorage, NodesContainer, Storage as IStorage, StorageGetter, ObjId, NodeEvent, SyncedObjStatus, DownloadEventSink } from '../../../lib-client/xsp-fs/common';
3
3
  import { ScryptGenParams } from '../../../lib-client/key-derivation';
4
4
  import { LogError } from '../../../lib-client/logging/log-to-file';
5
5
  import { AsyncSBoxCryptor, Subscribe, ObjSource } from 'xsp-files';
@@ -12,7 +12,7 @@ type FileEvent = web3n.files.FileEvent;
12
12
  type SyncStatus = web3n.files.SyncStatus;
13
13
  type OptionsToAdopteRemote = web3n.files.OptionsToAdopteRemote;
14
14
  type UploadEvent = web3n.files.UploadEvent;
15
- export declare class SyncedStore implements ISyncedStorage {
15
+ export declare class SyncedStore implements SyncedStorage {
16
16
  private readonly files;
17
17
  private readonly remoteStorage;
18
18
  private readonly getStorages;
@@ -27,11 +27,11 @@ export declare class SyncedStore implements ISyncedStorage {
27
27
  private readonly events;
28
28
  private constructor();
29
29
  static makeAndStart(path: string, user: string, getSigner: IGetMailerIdSigner, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, random: AsyncRNG, remoteServiceUrl: () => Promise<string>, net: NetClient, logError: LogError): Promise<{
30
- syncedStore: ISyncedStorage;
30
+ syncedStore: SyncedStorage;
31
31
  startObjProcs: () => void;
32
32
  }>;
33
33
  static makeAndStartWithoutRemote(path: string, user: string, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, random: AsyncRNG, remoteServiceUrl: () => Promise<string>, net: NetClient, logError: LogError): Promise<{
34
- syncedStore: ISyncedStorage;
34
+ syncedStore: SyncedStorage;
35
35
  setupRemoteAndStartObjProcs: (getSigner: IGetMailerIdSigner) => void;
36
36
  }>;
37
37
  getNodeEvents(): Observable<NodeEvent>;
@@ -9,7 +9,6 @@ export declare class AppDataFolders {
9
9
  private syncP;
10
10
  private makeSyncedFolder;
11
11
  private syncBeforeChange;
12
- private uploadAfterCreationOf;
13
12
  private startSyncProc;
14
13
  stopSync(): void;
15
14
  }
@@ -77,7 +77,8 @@ class AppDataFolders {
77
77
  }
78
78
  }
79
79
  const fs = await this.fs.writableSubRoot(folder, { create: false });
80
- await this.uploadAfterCreationOf(folder);
80
+ await this.fs.v.sync.upload(folder);
81
+ await this.fs.v.sync.upload('');
81
82
  return fs;
82
83
  });
83
84
  }
@@ -87,11 +88,6 @@ class AppDataFolders {
87
88
  await this.fs.v.sync.adoptRemote('');
88
89
  }
89
90
  }
90
- async uploadAfterCreationOf(folder) {
91
- // XXX must add work with not-online condition
92
- await this.fs.v.sync.upload(folder);
93
- await this.fs.v.sync.upload('');
94
- }
95
91
  startSyncProc() {
96
92
  this.syncFolderProc = (new rxjs_1.Observable((obs) => this.fs.watchFolder('', obs)))
97
93
  .pipe((0, operators_1.filter)(ev => (ev.type === 'remote-change')), (0, operators_1.mergeMap)(() => this.syncP().startOrChain(() => this.fs.v.sync.adoptRemote('')), 1))
@@ -5,7 +5,7 @@ type RemoteEvent = web3n.files.RemoteEvent;
5
5
  type FileEvent = web3n.files.FileEvent;
6
6
  type FolderEvent = web3n.files.FolderEvent;
7
7
  export declare function getRemoteFolderChanges(fs: WritableFS): Promise<void>;
8
- export declare function getOrMakeAndUploadFolderIn(fs: WritableFS, folder: string): Promise<WritableFS>;
8
+ export declare function getOrMakeSyncedDirOnInit(syncedFS: WritableFS, dir: string): Promise<WritableFS>;
9
9
  export declare function uploadFolderChangesIfAny(fs: WritableFS): Promise<void>;
10
10
  export declare function observableFromTreeEvents(fs: ReadonlyFS, rootPath: string): Observable<RemoteEvent | FileEvent | FolderEvent>;
11
11
  export {};
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  /*
3
- Copyright (C) 2022, 2025 3NSoft Inc.
3
+ Copyright (C) 2022, 2025 - 2026 3NSoft Inc.
4
4
 
5
5
  This program is free software: you can redistribute it and/or modify it under
6
6
  the terms of the GNU General Public License as published by the Free Software
@@ -17,11 +17,12 @@
17
17
  */
18
18
  Object.defineProperty(exports, "__esModule", { value: true });
19
19
  exports.getRemoteFolderChanges = getRemoteFolderChanges;
20
- exports.getOrMakeAndUploadFolderIn = getOrMakeAndUploadFolderIn;
20
+ exports.getOrMakeSyncedDirOnInit = getOrMakeSyncedDirOnInit;
21
21
  exports.uploadFolderChangesIfAny = uploadFolderChangesIfAny;
22
22
  exports.observableFromTreeEvents = observableFromTreeEvents;
23
23
  const rxjs_1 = require("rxjs");
24
24
  const exceptions_1 = require("../xsp-fs/exceptions");
25
+ const assert_1 = require("../../lib-common/assert");
25
26
  async function getRemoteFolderChanges(fs) {
26
27
  let { state } = await fs.v.sync.status('');
27
28
  if (state === 'behind') {
@@ -35,23 +36,79 @@ async function getRemoteFolderChanges(fs) {
35
36
  });
36
37
  }
37
38
  }
38
- async function getOrMakeAndUploadFolderIn(fs, folder) {
39
+ async function getOrMakeSyncedDirOnInit(syncedFS, dir) {
40
+ // Except for initial time, all folders exist and synced into parent, so, we shortcut sync check, if dir exists.
41
+ // This allows to start while offline, when initialization has been done.
42
+ if (await syncedFS.checkFolderPresence(dir)) {
43
+ return await syncedFS.writableSubRoot(dir, { create: false });
44
+ }
45
+ await ensureIsInSyncedState(syncedFS);
46
+ if (await syncedFS.checkFolderPresence(dir)) {
47
+ return await syncedFS.writableSubRoot(dir, { create: false });
48
+ }
39
49
  try {
40
- const childFolder = await fs.writableSubRoot(folder, { create: false });
41
- return childFolder;
50
+ const fs = await syncedFS.writableSubRoot(dir);
51
+ await syncedFS.v.sync.upload(dir);
52
+ await syncedFS.v.sync.upload('.');
53
+ return fs;
42
54
  }
43
- catch (exc) {
44
- if ((exc.type === 'file')
45
- && exc.notFound) {
46
- const childFolder = await fs.writableSubRoot(folder);
47
- await fs.v.sync.upload(folder);
48
- return childFolder;
55
+ catch (err) {
56
+ await syncedFS.v.sync.adoptRemote('.').catch(noop);
57
+ throw err;
58
+ }
59
+ }
60
+ function noop() { }
61
+ async function ensureIsInSyncedState(syncedFS) {
62
+ const { existsInSyncedParent, state, remote } = await syncedFS.v.sync.status('');
63
+ // assert(!!existsInSyncedParent, `${syncedFS.name} is not present in synced parent, preventing further construction of synced fs tree`);
64
+ if (state === 'synced') {
65
+ return;
66
+ }
67
+ else if (state === 'behind') {
68
+ await syncedFS.v.sync.adoptRemote('.');
69
+ return;
70
+ }
71
+ else if (state === 'unsynced') {
72
+ await syncedFS.v.sync.upload('.').catch(async (exc) => {
73
+ if (exc.type !== 'fs-sync') {
74
+ throw exc;
75
+ }
76
+ else if (exc.childNeverUploaded) {
77
+ await recursivelyUploadNewTreeItems(syncedFS, '.');
78
+ }
79
+ });
80
+ return;
81
+ }
82
+ (0, assert_1.assert)(state !== 'conflicting');
83
+ }
84
+ async function recursivelyUploadNewTreeItems(fs, path) {
85
+ for (const item of await fs.listFolder(path)) {
86
+ const { remote, synced } = await fs.v.sync.status(`${path}/${item.name}`);
87
+ if (remote || synced) {
88
+ continue;
49
89
  }
50
- else {
51
- throw exc;
90
+ if (item.isFolder) {
91
+ await recursivelyUploadNewTreeItems(fs, `${path}/${item.name}`);
52
92
  }
93
+ await fs.v.sync.upload(`${path}/${item.name}`);
53
94
  }
95
+ await fs.v.sync.upload(path);
54
96
  }
97
+ // export async function getOrMakeAndUploadFolderIn(fs: WritableFS, folder: string): Promise<WritableFS> {
98
+ // try {
99
+ // const childFolder = await fs.writableSubRoot(folder, { create: false });
100
+ // return childFolder;
101
+ // } catch (exc) {
102
+ // if (((exc as FileException).type === 'file') && (exc as FileException).notFound) {
103
+ // const childFolder = await fs.writableSubRoot(folder);
104
+ // await fs.v!.sync!.upload(folder);
105
+ // await fs.v!.sync!.upload('.');
106
+ // return childFolder;
107
+ // } else {
108
+ // throw exc;
109
+ // }
110
+ // }
111
+ // }
55
112
  // XXX conflicts are app(let)-specific, hence, we can't have this "general" functionality.
56
113
  async function uploadFolderChangesIfAny(fs) {
57
114
  try {
@@ -530,7 +530,9 @@ class V {
530
530
  async getOrCreateFile(path, flags) {
531
531
  const { fileName, folderPath } = split(path);
532
532
  const { create, exclusive } = flags;
533
- const folder = await this.getRootIfNotClosed(path).getFolderInThisSubTree(folderPath, create).catch(setExcPath(path));
533
+ const folder = await this.getRootIfNotClosed(path)
534
+ .getFolderInThisSubTree(folderPath, create)
535
+ .catch(setExcPath(path));
534
536
  const nullOnMissing = create;
535
537
  let file = await folder.getFile(fileName, nullOnMissing)
536
538
  .catch(setExcPath(path));
@@ -81,6 +81,7 @@ export declare abstract class NodeInFS<P extends NodePersistance> implements Nod
81
81
  get event$(): Observable<FSEvent | RemoteEvent>;
82
82
  protected syncedStorage(): SyncedStorage;
83
83
  syncStatus(): Promise<SyncStatus>;
84
+ private isThisPresentInSyncedParent;
84
85
  updateStatusInfo(): Promise<SyncStatus>;
85
86
  isSyncedVersionOnDisk(version: number): Promise<'partial' | 'complete' | 'none'>;
86
87
  startDownload(version: number): Promise<{
@@ -344,17 +344,22 @@ class NodeInFS {
344
344
  async syncStatus() {
345
345
  const storage = this.syncedStorage();
346
346
  const status = (await storage.status(this.objId)).syncStatus();
347
+ status.existsInSyncedParent = await this.isThisPresentInSyncedParent();
348
+ return status;
349
+ }
350
+ async isThisPresentInSyncedParent() {
347
351
  if (this.parentId) {
352
+ const storage = this.syncedStorage();
348
353
  const parent = storage.nodes.get(this.parentId);
349
354
  if (parent) {
350
- status.existsInSyncedParent = await parent.childExistsInSyncedVersion(this.objId);
355
+ return await parent.childExistsInSyncedVersion(this.objId);
351
356
  }
352
357
  }
353
- return status;
354
358
  }
355
359
  async updateStatusInfo() {
356
360
  const storage = this.syncedStorage();
357
361
  const status = await storage.updateStatusInfo(this.objId);
362
+ status.existsInSyncedParent = await this.isThisPresentInSyncedParent();
358
363
  return status;
359
364
  }
360
365
  isSyncedVersionOnDisk(version) {
@@ -137,9 +137,7 @@ class NodePersistance {
137
137
  };
138
138
  }
139
139
  async writableSink(newVersion, attrs, xattrs, base) {
140
- const segWriter = await (base ?
141
- this.segWriterWithBase(newVersion, base) :
142
- this.segWriter(newVersion));
140
+ const segWriter = await (base ? this.segWriterWithBase(newVersion, base) : this.segWriter(newVersion));
143
141
  const { sink, sub } = (0, xsp_files_1.makeEncryptingByteSink)(segWriter);
144
142
  const defSink = (0, deferred_1.defer)();
145
143
  return {
@@ -605,8 +605,7 @@ class WritablePayloadV2 {
605
605
  if (this.sectionsInSink.length === 0) {
606
606
  return;
607
607
  }
608
- let fstNoCutSectionInd = this.sectionsInSink
609
- .findIndex(s => (s.ofsInSrc >= this.cutonlyAreaEnd));
608
+ let fstNoCutSectionInd = this.sectionsInSink.findIndex(s => (s.ofsInSrc >= this.cutonlyAreaEnd));
610
609
  if (fstNoCutSectionInd < 0) {
611
610
  fstNoCutSectionInd = this.sectionsInSink.length;
612
611
  }
@@ -0,0 +1,58 @@
1
+ import { SingleProc } from '../processes/synced';
2
+ type WritableFS = web3n.files.WritableFS;
3
+ /**
4
+ * Log entry in synced files that act as broadcast of what operation a particular device has done.
5
+ * There may be race condition, in which devices may perform and record same operation simultaneously in different
6
+ * log files.
7
+ * There will also be situations when operation is recorded only on one device, like when triggered by action on
8
+ * one device.
9
+ *
10
+ * Regular log entry has payload in `p` and `n` with entry number that is an increment counter within a given log.
11
+ * When data set is synced to server, under some version `v`, a log entry with `syncPoint` is added.
12
+ * `syncPoint` contains `logSyncedPoints` with last operations on known devices that have been incorporated into
13
+ * the data set synced state of given version.
14
+ *
15
+ * Sync point information allows to ignore log entries, cutting log files accordingly.
16
+ */
17
+ export interface LogEntry<T> {
18
+ /**
19
+ * When data set is synced to server, under some version `v`, this log entry is added.
20
+ * This contains `logSyncedPoints` with last operations on known devices that have been incorporated into data
21
+ * set synced state.
22
+ */
23
+ syncPoint?: {
24
+ v: number;
25
+ logSyncedPoints: {
26
+ [devId: string]: number;
27
+ };
28
+ params: any;
29
+ };
30
+ /**
31
+ * Increasing log entry number, unique to a given device log.
32
+ */
33
+ n: number;
34
+ /**
35
+ * Timestamp of log helps to somewhat order logs from many device.
36
+ */
37
+ ts: number;
38
+ /**
39
+ * Payload of regular log entry.
40
+ */
41
+ p?: T;
42
+ }
43
+ export declare function makeRecentChangesLogs<T>(logsFS: WritableFS, localLogsFS: WritableFS, areOpsEqual: (a: T, b: T) => boolean, logError: (err: any, msg?: string) => Promise<void>): Promise<{
44
+ appendLogsAndUpload: (...ops: T[]) => Promise<void> | undefined;
45
+ watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, sink: (ops: LogEntry<T>[]) => Promise<void>) => () => void;
46
+ numOfLogs: () => number;
47
+ cutLogOnDatasetSyncAndUpload: (syncPointV: number, params?: any) => Promise<void> | undefined;
48
+ }>;
49
+ export type RecentChangesLogs<T> = Awaited<ReturnType<typeof makeRecentChangesLogs<T>>>;
50
+ export interface DatasetInstance<T> {
51
+ watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, resetSyncInterval: () => void) => (() => void);
52
+ absorbOpsFromOtherDevices: (changeProc: SingleProc, ops: LogEntry<T>[]) => Promise<void>;
53
+ }
54
+ export declare function startDatasetSync<T>(periodicCheckAndUploadOfData: () => Promise<void>, millisBeforeNextRun: () => number, logs: Pick<RecentChangesLogs<T>, 'watchAndApplyOpsFromOtherDevices'>, dataset: DatasetInstance<T>): {
55
+ changeProc: SingleProc;
56
+ stopSyncing: () => void;
57
+ };
58
+ export {};