core-3nweb-client-lib 0.49.5 → 0.49.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/api-defs/files.d.ts +11 -5
- package/build/core/asmail/delivery/common.d.ts +8 -28
- package/build/core/asmail/delivery/common.js +1 -47
- package/build/core/asmail/delivery/index.js +3 -8
- package/build/core/asmail/delivery/msg.js +1 -7
- package/build/core/asmail/delivery/per-recipient-wip.js +3 -29
- package/build/core/asmail/inbox/index.d.ts +1 -2
- package/build/core/asmail/inbox/index.js +14 -15
- package/build/core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.d.ts +11 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.js +25 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.d.ts +33 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.js +72 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.d.ts +28 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.js +477 -0
- package/build/core/asmail/inbox/msg-indexing/index.d.ts +12 -26
- package/build/core/asmail/inbox/msg-indexing/index.js +47 -68
- package/build/core/asmail/inbox/msg-on-disk.js +1 -2
- package/build/core/asmail/index.js +8 -7
- package/build/core/asmail/msg/common.d.ts +0 -1
- package/build/core/asmail/msg/common.js +1 -1
- package/build/core/asmail/msg/packer.js +1 -3
- package/build/core/index.js +7 -4
- package/build/core/keyring/common.d.ts +1 -0
- package/build/core/keyring/common.js +2 -1
- package/build/core/keyring/index.d.ts +7 -44
- package/build/core/keyring/index.js +89 -222
- package/build/core/keyring/introductory-keys/published-intro-key.d.ts +17 -0
- package/build/core/keyring/introductory-keys/published-intro-key.js +190 -0
- package/build/core/keyring/peer-keys/dataset/v1/json-files.d.ts +105 -0
- package/build/core/keyring/peer-keys/dataset/v1/json-files.js +89 -0
- package/build/core/keyring/peer-keys/dataset/v2/logs.d.ts +43 -0
- package/build/core/keyring/peer-keys/dataset/v2/logs.js +116 -0
- package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.d.ts +73 -0
- package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.js +543 -0
- package/build/core/keyring/peer-keys/index.d.ts +24 -0
- package/build/core/keyring/peer-keys/index.js +208 -0
- package/build/core/storage/synced/obj-files.js +10 -3
- package/build/core/storage/synced/obj-status.d.ts +1 -1
- package/build/core/storage/synced/obj-status.js +8 -2
- package/build/core/storage/synced/storage.d.ts +4 -4
- package/build/core/storage/system-folders/apps-data.d.ts +0 -1
- package/build/core/storage/system-folders/apps-data.js +2 -6
- package/build/lib-client/fs-utils/fs-sync-utils.d.ts +1 -1
- package/build/lib-client/fs-utils/fs-sync-utils.js +70 -13
- package/build/lib-client/xsp-fs/fs.js +3 -1
- package/build/lib-client/xsp-fs/node-in-fs.d.ts +1 -0
- package/build/lib-client/xsp-fs/node-in-fs.js +7 -2
- package/build/lib-client/xsp-fs/node-persistence.js +1 -3
- package/build/lib-client/xsp-fs/xsp-payload-v2.js +1 -2
- package/build/lib-common/dataset-sync/per-device-json-array-logs.d.ts +58 -0
- package/build/lib-common/dataset-sync/per-device-json-array-logs.js +432 -0
- package/build/lib-common/number-line.d.ts +10 -0
- package/build/lib-common/number-line.js +66 -0
- package/build/lib-common/objs-on-disk/obj-version-file.d.ts +1 -1
- package/build/lib-common/objs-on-disk/obj-version-file.js +3 -3
- package/build/lib-common/rng-def.d.ts +1 -0
- package/build/lib-common/rng-def.js +8 -0
- package/build/lib-sqlite-on-3nstorage/for-sqlite.d.ts +38 -0
- package/build/lib-sqlite-on-3nstorage/for-sqlite.js +162 -0
- package/build/lib-sqlite-on-3nstorage/index.d.ts +18 -5
- package/build/lib-sqlite-on-3nstorage/index.js +49 -26
- package/build/tests/caps-api/file-sink-checks/splice.js +72 -1
- package/build/tests/caps-api/fs-checks/sync-on-one-dev/upload.js +27 -1
- package/build/tests/caps-api/fs-checks/sync-with-two-devs/update-propagation.js +12 -3
- package/build/tests/computer.3nweb.core/inbox/msg-indexing.js +16 -12
- package/build/tests/computer.3nweb.core/keyrings.js +113 -28
- package/build/tests/jasmine.js +2 -2
- package/build/tests/units/number-line.js +9 -9
- package/package.json +2 -1
- package/build/core/asmail/inbox/msg-indexing/logs-n-entries.d.ts +0 -52
- package/build/core/asmail/inbox/msg-indexing/logs-n-entries.js +0 -333
- package/build/core/asmail/inbox/msg-indexing/sql-indexing.d.ts +0 -4
- package/build/core/asmail/inbox/msg-indexing/sql-indexing.js +0 -319
- package/build/core/keyring/correspondent-keys.d.ts +0 -149
- package/build/core/keyring/correspondent-keys.js +0 -403
- package/build/core/keyring/id-to-email-map.d.ts +0 -38
- package/build/core/keyring/id-to-email-map.js +0 -105
- package/build/core/keyring/keyring-storage.d.ts +0 -9
- package/build/core/keyring/keyring-storage.js +0 -54
- package/build/core/keyring/published-intro-key.d.ts +0 -45
- package/build/core/keyring/published-intro-key.js +0 -200
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2015 - 2018, 2022, 2026 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.makePeersKeyring = makePeersKeyring;
|
|
20
|
+
const common_1 = require("../common");
|
|
21
|
+
const ecma_nacl_1 = require("ecma-nacl");
|
|
22
|
+
const buffer_utils_1 = require("../../../lib-common/buffer-utils");
|
|
23
|
+
const fs_sync_utils_1 = require("../../../lib-client/fs-utils/fs-sync-utils");
|
|
24
|
+
const per_device_json_array_logs_1 = require("../../../lib-common/dataset-sync/per-device-json-array-logs");
|
|
25
|
+
const synced_1 = require("../../../lib-common/processes/synced");
|
|
26
|
+
const logs_1 = require("./dataset/v2/logs");
|
|
27
|
+
const peer_keys_db_1 = require("./dataset/v2/peer-keys-db");
|
|
28
|
+
const canonical_address_1 = require("../../../lib-common/canonical-address");
|
|
29
|
+
const LOGS_DIR = 'peer-keys-logs';
|
|
30
|
+
const PEER_KEYS_FOLDER = 'peer-keys-db';
|
|
31
|
+
const NUM_OF_LOGS_TO_START_CUT = 200;
|
|
32
|
+
async function makePeersKeyring(keyringFS, localFS, random, logger) {
|
|
33
|
+
const logsFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(keyringFS, LOGS_DIR);
|
|
34
|
+
const logsLocalFS = await localFS.writableSubRoot(LOGS_DIR);
|
|
35
|
+
const peerKeysFS = await (0, fs_sync_utils_1.getOrMakeSyncedDirOnInit)(keyringFS, PEER_KEYS_FOLDER);
|
|
36
|
+
// XXX log and saving reduction(!):
|
|
37
|
+
// - we don't need to log message counts
|
|
38
|
+
// - we don't have to immediately save changes in message counts
|
|
39
|
+
// - we want to save and log crypto material, more so, we generate and log new crypto first, then send message
|
|
40
|
+
// - logging and saving of marking pair as in use also doesn't have to be immediate (should it be logged?)
|
|
41
|
+
const logs = await (0, logs_1.makePeerKeysChangeLogs)(logsFS, logsLocalFS, logger.logError);
|
|
42
|
+
const keysDB = await (0, peer_keys_db_1.makePeerKeysDB)(peerKeysFS, random);
|
|
43
|
+
async function suggestPairForPeerIntroKey(peerAddr, peerIntroPKey) {
|
|
44
|
+
const peerCAddr = (0, canonical_address_1.toCanonicalAddress)(peerAddr);
|
|
45
|
+
const alreadySuggested = keysDB.getPairSuggestedToPeer(peerCAddr);
|
|
46
|
+
if (alreadySuggested && (alreadySuggested.senderKid === peerIntroPKey.kid)) {
|
|
47
|
+
return alreadySuggested;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
if (!keysDB.getPeerAddressForCanonical(peerCAddr)) {
|
|
51
|
+
keysDB.addPeer(peerAddr);
|
|
52
|
+
}
|
|
53
|
+
const { nextCrypto, entryToLog } = await keysDB.generateSuggestedPairOnPeerIntroKey(peerCAddr, peerIntroPKey);
|
|
54
|
+
await keysDB.saveIfNeeded();
|
|
55
|
+
await logs.recordSuggestingCryptoToPeer(entryToLog, peerAddr);
|
|
56
|
+
return nextCrypto;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function needIntroKeyFor(peerAddr) {
|
|
60
|
+
return !keysDB.getSendingPair((0, canonical_address_1.toCanonicalAddress)(peerAddr));
|
|
61
|
+
}
|
|
62
|
+
async function getSendingCryptoWithinEstablishedPair(peerCAddr) {
|
|
63
|
+
const pair = keysDB.getSendingPair(peerCAddr);
|
|
64
|
+
if (!pair) {
|
|
65
|
+
throw new Error(`No sending pair for peer ${peerCAddr}`);
|
|
66
|
+
}
|
|
67
|
+
const { pids, peerKId, pairAlg, senderKId, senderPKey, msgMasterKey, sentMsgsCount } = pair;
|
|
68
|
+
const msgCount = sentMsgsCount + 1;
|
|
69
|
+
keysDB.updateSentMsgCountInSendingPair(peerCAddr, msgCount);
|
|
70
|
+
let nextMsgCrypto = keysDB.getPairSuggestedToPeer(peerCAddr);
|
|
71
|
+
if (!nextMsgCrypto) {
|
|
72
|
+
const generated = await keysDB.generateRegularSuggestedPairIfNeeded(peerCAddr);
|
|
73
|
+
if (generated) {
|
|
74
|
+
nextMsgCrypto = generated.nextCrypto;
|
|
75
|
+
await keysDB.saveIfNeeded();
|
|
76
|
+
await logs.recordSuggestingCryptoToPeer(generated.entryToLog, peerCAddr);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
currentPair: {
|
|
81
|
+
pid: selectPid(pids),
|
|
82
|
+
recipientKid: peerKId,
|
|
83
|
+
senderPKey: {
|
|
84
|
+
alg: pairAlg,
|
|
85
|
+
kid: senderKId,
|
|
86
|
+
k: senderPKey,
|
|
87
|
+
use: common_1.KEY_USE.PUBLIC
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
msgMasterKey,
|
|
91
|
+
msgCount,
|
|
92
|
+
nextMsgCrypto
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
async function markPairAsInUse(peerCAddr, peerKId, recipientKId) {
|
|
96
|
+
const pairFoundAndMarked = keysDB.markPairAsInUse(peerCAddr, peerKId, recipientKId);
|
|
97
|
+
if (pairFoundAndMarked) {
|
|
98
|
+
await keysDB.saveIfNeeded();
|
|
99
|
+
await logs.recordPeerStartedUsingPair(peerCAddr, peerKId, recipientKId);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
async function absorbNextPairSuggestedByPeer(peerAddr, pair, introKey) {
|
|
103
|
+
const peerCAddr = (0, canonical_address_1.toCanonicalAddress)(peerAddr);
|
|
104
|
+
const msgMasterKeyAlg = ecma_nacl_1.box.JWK_ALG_NAME;
|
|
105
|
+
const peerPKey = buffer_utils_1.base64.open(pair.recipientPKey.k);
|
|
106
|
+
if (keysDB.getPeerAddressForCanonical(peerCAddr)) {
|
|
107
|
+
const prevPair = keysDB.getSendingPair(peerCAddr);
|
|
108
|
+
if (introKey) {
|
|
109
|
+
if (prevPair
|
|
110
|
+
&& ((introKey.pkey.kid === pair.senderKid) && (prevPair.peerKId === pair.recipientPKey.kid))) {
|
|
111
|
+
// ignoring duplicate
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const senderSKey = buffer_utils_1.base64.open(introKey.skey.k);
|
|
115
|
+
const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
|
|
116
|
+
keysDB.updateSendingPairFromSuggested(peerCAddr, pair, {
|
|
117
|
+
msgMasterKey, msgMasterKeyAlg, peerPKey, senderSKey,
|
|
118
|
+
senderKId: introKey.pkey.kid,
|
|
119
|
+
senderPKey: introKey.pkey.k
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
if (prevPair
|
|
124
|
+
&& ((prevPair.senderKId === pair.senderKid) && (prevPair.peerKId === pair.recipientPKey.kid))) {
|
|
125
|
+
// ignoring duplicate
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const foundPair = keysDB.findKeyToMatchSuggestedForSendingPair(peerCAddr, pair.senderKid);
|
|
129
|
+
if (!foundPair) {
|
|
130
|
+
// ignore bad
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
const { pkey: senderPKey, skey: senderSKey, pairAlg } = foundPair;
|
|
134
|
+
const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
|
|
135
|
+
if (prevPair) {
|
|
136
|
+
keysDB.updateSendingPairFromSuggested(peerCAddr, pair, {
|
|
137
|
+
msgMasterKey, msgMasterKeyAlg, peerPKey, senderSKey, senderPKey,
|
|
138
|
+
senderKId: pair.senderKid
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
keysDB.addSendingPairFromSuggested(peerCAddr, pair, {
|
|
143
|
+
msgMasterKey, msgMasterKeyAlg, senderSKey, peerPKey, pairAlg, senderPKey,
|
|
144
|
+
senderKId: pair.senderKid
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
if (!introKey) {
|
|
151
|
+
throw new Error(`Peer ${peerAddr} is not known, while used introductory key is not given`);
|
|
152
|
+
}
|
|
153
|
+
keysDB.addPeer(peerAddr);
|
|
154
|
+
const senderSKey = buffer_utils_1.base64.open(introKey.skey.k);
|
|
155
|
+
const msgMasterKey = ecma_nacl_1.box.calc_dhshared_key(peerPKey, senderSKey);
|
|
156
|
+
keysDB.addSendingPairFromSuggested(peerCAddr, pair, {
|
|
157
|
+
msgMasterKey, msgMasterKeyAlg, senderSKey, peerPKey,
|
|
158
|
+
pairAlg: introKey.pkey.alg,
|
|
159
|
+
senderKId: introKey.pkey.kid,
|
|
160
|
+
senderPKey: introKey.pkey.k
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
const entryToLog = keysDB.getSendingPair(peerCAddr);
|
|
164
|
+
await keysDB.saveIfNeeded();
|
|
165
|
+
await logs.recordGettingCryptoFromPeer(entryToLog, peerAddr);
|
|
166
|
+
}
|
|
167
|
+
async function periodicCheckAndUploadOfData() {
|
|
168
|
+
// XXX disable uploading for a moment by commenting out
|
|
169
|
+
// if (logs.numOfLogs() >= NUM_OF_LOGS_TO_START_CUT) {
|
|
170
|
+
// const syncedIndexDbVersion = await keysDB.saveAndSync();
|
|
171
|
+
// await logs.cutLogOnDatasetSyncAndUpload(syncedIndexDbVersion);
|
|
172
|
+
// }
|
|
173
|
+
}
|
|
174
|
+
function millisBeforeNextRun() {
|
|
175
|
+
return 30 * 60 * 1000 + Math.floor(10 * 60 * 1000 * Math.random());
|
|
176
|
+
}
|
|
177
|
+
const { stopSyncing, changeProc } = (0, per_device_json_array_logs_1.startDatasetSync)(periodicCheckAndUploadOfData, millisBeforeNextRun, {
|
|
178
|
+
watchAndApplyOpsFromOtherDevices: logs.watchAndApplyOpsFromOtherDevices
|
|
179
|
+
}, {
|
|
180
|
+
absorbOpsFromOtherDevices: keysDB.absorbOpsFromOtherDevices,
|
|
181
|
+
watchAndApplyOpsFromOtherDevices: keysDB.watchAndApplyOpsFromOtherDevices
|
|
182
|
+
});
|
|
183
|
+
// XXX
|
|
184
|
+
// - check if indexed latest is too old, chopping to the latest shard
|
|
185
|
+
async function close() {
|
|
186
|
+
stopSyncing();
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
getPeerAddressForCanonical: keysDB.getPeerAddressForCanonical,
|
|
190
|
+
needIntroKeyFor,
|
|
191
|
+
suggestPairForPeerIntroKey: (0, synced_1.makeSyncedFunc)(changeProc, undefined, suggestPairForPeerIntroKey),
|
|
192
|
+
getSendingCryptoWithinEstablishedPair: (0, synced_1.makeSyncedFunc)(changeProc, undefined, getSendingCryptoWithinEstablishedPair),
|
|
193
|
+
absorbNextPairSuggestedByPeer: (0, synced_1.makeSyncedFunc)(changeProc, undefined, absorbNextPairSuggestedByPeer),
|
|
194
|
+
findEstablishedReceptionPairs: keysDB.findReceptionPairsForPId,
|
|
195
|
+
markPairAsInUse: (0, synced_1.makeSyncedFunc)(changeProc, undefined, markPairAsInUse),
|
|
196
|
+
updateReceivedMsgCountIn: keysDB.updateReceivedMsgCountIn,
|
|
197
|
+
close,
|
|
198
|
+
getPeerKeysInfo: keysDB.getPeerKeysInfo
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
function selectPid(pids) {
|
|
202
|
+
if (pids.length < 1) {
|
|
203
|
+
throw new Error("There are no pair ids in array.");
|
|
204
|
+
}
|
|
205
|
+
const i = Math.round((pids.length - 1) * Math.random());
|
|
206
|
+
return pids[i];
|
|
207
|
+
}
|
|
208
|
+
Object.freeze(exports);
|
|
@@ -231,7 +231,14 @@ class SyncedObj {
|
|
|
231
231
|
const status = await obj_status_1.ObjStatus.makeForDownloadedVersion(objFolder, objId, version, version, logError);
|
|
232
232
|
const obj = new SyncedObj(objId, objFolder, status, downloader, scheduleGC);
|
|
233
233
|
const fPath = obj.remoteVerPath(version);
|
|
234
|
-
const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk, parts)
|
|
234
|
+
const objVer = await obj_on_disk_1.ObjOnDisk.createFileForExistingVersion(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk, parts).catch((exc) => {
|
|
235
|
+
if (exc.alreadyExists) {
|
|
236
|
+
return obj_on_disk_1.ObjOnDisk.forExistingFile(obj.objId, version, fPath, obj.downloader, obj.remoteObjSegsGetterFromDisk);
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
throw exc;
|
|
240
|
+
}
|
|
241
|
+
});
|
|
235
242
|
obj.remoteVers.set(version, objVer);
|
|
236
243
|
return obj;
|
|
237
244
|
}
|
|
@@ -397,7 +404,7 @@ class SyncedObj {
|
|
|
397
404
|
this.scheduleSelfGC();
|
|
398
405
|
}
|
|
399
406
|
async isRemoteVersionOnDisk(version) {
|
|
400
|
-
if (!this.status.
|
|
407
|
+
if (!this.status.isAmongRemoteOrSynced(version)) {
|
|
401
408
|
throw (0, exceptions_1.makeObjVersionNotFoundExc)(this.objId, version);
|
|
402
409
|
}
|
|
403
410
|
const verPath = this.remoteVerPath(version);
|
|
@@ -408,7 +415,7 @@ class SyncedObj {
|
|
|
408
415
|
return (objVer.doesFileNeedDownload() ? 'partial' : 'complete');
|
|
409
416
|
}
|
|
410
417
|
async getNumOfBytesNeedingDownload(version) {
|
|
411
|
-
if (!this.status.
|
|
418
|
+
if (!this.status.isAmongRemoteOrSynced(version)) {
|
|
412
419
|
if (this.status.isLocalOnlyVersion(version)) {
|
|
413
420
|
return 0;
|
|
414
421
|
}
|
|
@@ -147,7 +147,7 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
|
|
|
147
147
|
neverUploaded(): boolean;
|
|
148
148
|
versionBeforeUnsyncedRemoval(): number | undefined;
|
|
149
149
|
adoptRemoteVersion(version?: number): Promise<void>;
|
|
150
|
-
|
|
150
|
+
isAmongRemoteOrSynced(version: number): boolean;
|
|
151
151
|
isLocalOnlyVersion(version: number): boolean;
|
|
152
152
|
}
|
|
153
153
|
export declare function readAndCheckStatus(objFolder: string, objId: ObjId): Promise<ObjStatusInfo>;
|
|
@@ -553,14 +553,20 @@ class ObjStatus {
|
|
|
553
553
|
this.updateStateIndicator();
|
|
554
554
|
await this.triggerSaveProc();
|
|
555
555
|
}
|
|
556
|
-
|
|
557
|
-
var _a;
|
|
556
|
+
isAmongRemoteOrSynced(version) {
|
|
557
|
+
var _a, _b;
|
|
558
558
|
if (this.status.remote.current === version) {
|
|
559
559
|
return true;
|
|
560
560
|
}
|
|
561
561
|
if ((_a = this.status.remote.archived) === null || _a === void 0 ? void 0 : _a.includes(version)) {
|
|
562
562
|
return true;
|
|
563
563
|
}
|
|
564
|
+
if (this.status.remote.baseToDiff[version]) {
|
|
565
|
+
return true;
|
|
566
|
+
}
|
|
567
|
+
if (((_b = this.status.synced) === null || _b === void 0 ? void 0 : _b.version) === version) {
|
|
568
|
+
return true;
|
|
569
|
+
}
|
|
564
570
|
return false;
|
|
565
571
|
}
|
|
566
572
|
isLocalOnlyVersion(version) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { IGetMailerIdSigner } from '../../../lib-client/user-with-mid-session';
|
|
2
|
-
import { SyncedStorage
|
|
2
|
+
import { SyncedStorage, NodesContainer, Storage as IStorage, StorageGetter, ObjId, NodeEvent, SyncedObjStatus, DownloadEventSink } from '../../../lib-client/xsp-fs/common';
|
|
3
3
|
import { ScryptGenParams } from '../../../lib-client/key-derivation';
|
|
4
4
|
import { LogError } from '../../../lib-client/logging/log-to-file';
|
|
5
5
|
import { AsyncSBoxCryptor, Subscribe, ObjSource } from 'xsp-files';
|
|
@@ -12,7 +12,7 @@ type FileEvent = web3n.files.FileEvent;
|
|
|
12
12
|
type SyncStatus = web3n.files.SyncStatus;
|
|
13
13
|
type OptionsToAdopteRemote = web3n.files.OptionsToAdopteRemote;
|
|
14
14
|
type UploadEvent = web3n.files.UploadEvent;
|
|
15
|
-
export declare class SyncedStore implements
|
|
15
|
+
export declare class SyncedStore implements SyncedStorage {
|
|
16
16
|
private readonly files;
|
|
17
17
|
private readonly remoteStorage;
|
|
18
18
|
private readonly getStorages;
|
|
@@ -27,11 +27,11 @@ export declare class SyncedStore implements ISyncedStorage {
|
|
|
27
27
|
private readonly events;
|
|
28
28
|
private constructor();
|
|
29
29
|
static makeAndStart(path: string, user: string, getSigner: IGetMailerIdSigner, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, random: AsyncRNG, remoteServiceUrl: () => Promise<string>, net: NetClient, logError: LogError): Promise<{
|
|
30
|
-
syncedStore:
|
|
30
|
+
syncedStore: SyncedStorage;
|
|
31
31
|
startObjProcs: () => void;
|
|
32
32
|
}>;
|
|
33
33
|
static makeAndStartWithoutRemote(path: string, user: string, getStorages: StorageGetter, cryptor: AsyncSBoxCryptor, random: AsyncRNG, remoteServiceUrl: () => Promise<string>, net: NetClient, logError: LogError): Promise<{
|
|
34
|
-
syncedStore:
|
|
34
|
+
syncedStore: SyncedStorage;
|
|
35
35
|
setupRemoteAndStartObjProcs: (getSigner: IGetMailerIdSigner) => void;
|
|
36
36
|
}>;
|
|
37
37
|
getNodeEvents(): Observable<NodeEvent>;
|
|
@@ -77,7 +77,8 @@ class AppDataFolders {
|
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
const fs = await this.fs.writableSubRoot(folder, { create: false });
|
|
80
|
-
await this.
|
|
80
|
+
await this.fs.v.sync.upload(folder);
|
|
81
|
+
await this.fs.v.sync.upload('');
|
|
81
82
|
return fs;
|
|
82
83
|
});
|
|
83
84
|
}
|
|
@@ -87,11 +88,6 @@ class AppDataFolders {
|
|
|
87
88
|
await this.fs.v.sync.adoptRemote('');
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
|
-
async uploadAfterCreationOf(folder) {
|
|
91
|
-
// XXX must add work with not-online condition
|
|
92
|
-
await this.fs.v.sync.upload(folder);
|
|
93
|
-
await this.fs.v.sync.upload('');
|
|
94
|
-
}
|
|
95
91
|
startSyncProc() {
|
|
96
92
|
this.syncFolderProc = (new rxjs_1.Observable((obs) => this.fs.watchFolder('', obs)))
|
|
97
93
|
.pipe((0, operators_1.filter)(ev => (ev.type === 'remote-change')), (0, operators_1.mergeMap)(() => this.syncP().startOrChain(() => this.fs.v.sync.adoptRemote('')), 1))
|
|
@@ -5,7 +5,7 @@ type RemoteEvent = web3n.files.RemoteEvent;
|
|
|
5
5
|
type FileEvent = web3n.files.FileEvent;
|
|
6
6
|
type FolderEvent = web3n.files.FolderEvent;
|
|
7
7
|
export declare function getRemoteFolderChanges(fs: WritableFS): Promise<void>;
|
|
8
|
-
export declare function
|
|
8
|
+
export declare function getOrMakeSyncedDirOnInit(syncedFS: WritableFS, dir: string): Promise<WritableFS>;
|
|
9
9
|
export declare function uploadFolderChangesIfAny(fs: WritableFS): Promise<void>;
|
|
10
10
|
export declare function observableFromTreeEvents(fs: ReadonlyFS, rootPath: string): Observable<RemoteEvent | FileEvent | FolderEvent>;
|
|
11
11
|
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/*
|
|
3
|
-
Copyright (C) 2022, 2025 3NSoft Inc.
|
|
3
|
+
Copyright (C) 2022, 2025 - 2026 3NSoft Inc.
|
|
4
4
|
|
|
5
5
|
This program is free software: you can redistribute it and/or modify it under
|
|
6
6
|
the terms of the GNU General Public License as published by the Free Software
|
|
@@ -17,11 +17,12 @@
|
|
|
17
17
|
*/
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.getRemoteFolderChanges = getRemoteFolderChanges;
|
|
20
|
-
exports.
|
|
20
|
+
exports.getOrMakeSyncedDirOnInit = getOrMakeSyncedDirOnInit;
|
|
21
21
|
exports.uploadFolderChangesIfAny = uploadFolderChangesIfAny;
|
|
22
22
|
exports.observableFromTreeEvents = observableFromTreeEvents;
|
|
23
23
|
const rxjs_1 = require("rxjs");
|
|
24
24
|
const exceptions_1 = require("../xsp-fs/exceptions");
|
|
25
|
+
const assert_1 = require("../../lib-common/assert");
|
|
25
26
|
async function getRemoteFolderChanges(fs) {
|
|
26
27
|
let { state } = await fs.v.sync.status('');
|
|
27
28
|
if (state === 'behind') {
|
|
@@ -35,23 +36,79 @@ async function getRemoteFolderChanges(fs) {
|
|
|
35
36
|
});
|
|
36
37
|
}
|
|
37
38
|
}
|
|
38
|
-
async function
|
|
39
|
+
async function getOrMakeSyncedDirOnInit(syncedFS, dir) {
|
|
40
|
+
// Except for initial time, all folders exist and synced into parent, so, we shortcut sync check, if dir exists.
|
|
41
|
+
// This allows to start while offline, when initialization has been done.
|
|
42
|
+
if (await syncedFS.checkFolderPresence(dir)) {
|
|
43
|
+
return await syncedFS.writableSubRoot(dir, { create: false });
|
|
44
|
+
}
|
|
45
|
+
await ensureIsInSyncedState(syncedFS);
|
|
46
|
+
if (await syncedFS.checkFolderPresence(dir)) {
|
|
47
|
+
return await syncedFS.writableSubRoot(dir, { create: false });
|
|
48
|
+
}
|
|
39
49
|
try {
|
|
40
|
-
const
|
|
41
|
-
|
|
50
|
+
const fs = await syncedFS.writableSubRoot(dir);
|
|
51
|
+
await syncedFS.v.sync.upload(dir);
|
|
52
|
+
await syncedFS.v.sync.upload('.');
|
|
53
|
+
return fs;
|
|
42
54
|
}
|
|
43
|
-
catch (
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
55
|
+
catch (err) {
|
|
56
|
+
await syncedFS.v.sync.adoptRemote('.').catch(noop);
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function noop() { }
|
|
61
|
+
async function ensureIsInSyncedState(syncedFS) {
|
|
62
|
+
const { existsInSyncedParent, state, remote } = await syncedFS.v.sync.status('');
|
|
63
|
+
// assert(!!existsInSyncedParent, `${syncedFS.name} is not present in synced parent, preventing further construction of synced fs tree`);
|
|
64
|
+
if (state === 'synced') {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
else if (state === 'behind') {
|
|
68
|
+
await syncedFS.v.sync.adoptRemote('.');
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
else if (state === 'unsynced') {
|
|
72
|
+
await syncedFS.v.sync.upload('.').catch(async (exc) => {
|
|
73
|
+
if (exc.type !== 'fs-sync') {
|
|
74
|
+
throw exc;
|
|
75
|
+
}
|
|
76
|
+
else if (exc.childNeverUploaded) {
|
|
77
|
+
await recursivelyUploadNewTreeItems(syncedFS, '.');
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
(0, assert_1.assert)(state !== 'conflicting');
|
|
83
|
+
}
|
|
84
|
+
async function recursivelyUploadNewTreeItems(fs, path) {
|
|
85
|
+
for (const item of await fs.listFolder(path)) {
|
|
86
|
+
const { remote, synced } = await fs.v.sync.status(`${path}/${item.name}`);
|
|
87
|
+
if (remote || synced) {
|
|
88
|
+
continue;
|
|
49
89
|
}
|
|
50
|
-
|
|
51
|
-
|
|
90
|
+
if (item.isFolder) {
|
|
91
|
+
await recursivelyUploadNewTreeItems(fs, `${path}/${item.name}`);
|
|
52
92
|
}
|
|
93
|
+
await fs.v.sync.upload(`${path}/${item.name}`);
|
|
53
94
|
}
|
|
95
|
+
await fs.v.sync.upload(path);
|
|
54
96
|
}
|
|
97
|
+
// export async function getOrMakeAndUploadFolderIn(fs: WritableFS, folder: string): Promise<WritableFS> {
|
|
98
|
+
// try {
|
|
99
|
+
// const childFolder = await fs.writableSubRoot(folder, { create: false });
|
|
100
|
+
// return childFolder;
|
|
101
|
+
// } catch (exc) {
|
|
102
|
+
// if (((exc as FileException).type === 'file') && (exc as FileException).notFound) {
|
|
103
|
+
// const childFolder = await fs.writableSubRoot(folder);
|
|
104
|
+
// await fs.v!.sync!.upload(folder);
|
|
105
|
+
// await fs.v!.sync!.upload('.');
|
|
106
|
+
// return childFolder;
|
|
107
|
+
// } else {
|
|
108
|
+
// throw exc;
|
|
109
|
+
// }
|
|
110
|
+
// }
|
|
111
|
+
// }
|
|
55
112
|
// XXX conflicts are app(let)-specific, hence, we can't have this "general" functionality.
|
|
56
113
|
async function uploadFolderChangesIfAny(fs) {
|
|
57
114
|
try {
|
|
@@ -530,7 +530,9 @@ class V {
|
|
|
530
530
|
async getOrCreateFile(path, flags) {
|
|
531
531
|
const { fileName, folderPath } = split(path);
|
|
532
532
|
const { create, exclusive } = flags;
|
|
533
|
-
const folder = await this.getRootIfNotClosed(path)
|
|
533
|
+
const folder = await this.getRootIfNotClosed(path)
|
|
534
|
+
.getFolderInThisSubTree(folderPath, create)
|
|
535
|
+
.catch(setExcPath(path));
|
|
534
536
|
const nullOnMissing = create;
|
|
535
537
|
let file = await folder.getFile(fileName, nullOnMissing)
|
|
536
538
|
.catch(setExcPath(path));
|
|
@@ -81,6 +81,7 @@ export declare abstract class NodeInFS<P extends NodePersistance> implements Nod
|
|
|
81
81
|
get event$(): Observable<FSEvent | RemoteEvent>;
|
|
82
82
|
protected syncedStorage(): SyncedStorage;
|
|
83
83
|
syncStatus(): Promise<SyncStatus>;
|
|
84
|
+
private isThisPresentInSyncedParent;
|
|
84
85
|
updateStatusInfo(): Promise<SyncStatus>;
|
|
85
86
|
isSyncedVersionOnDisk(version: number): Promise<'partial' | 'complete' | 'none'>;
|
|
86
87
|
startDownload(version: number): Promise<{
|
|
@@ -344,17 +344,22 @@ class NodeInFS {
|
|
|
344
344
|
async syncStatus() {
|
|
345
345
|
const storage = this.syncedStorage();
|
|
346
346
|
const status = (await storage.status(this.objId)).syncStatus();
|
|
347
|
+
status.existsInSyncedParent = await this.isThisPresentInSyncedParent();
|
|
348
|
+
return status;
|
|
349
|
+
}
|
|
350
|
+
async isThisPresentInSyncedParent() {
|
|
347
351
|
if (this.parentId) {
|
|
352
|
+
const storage = this.syncedStorage();
|
|
348
353
|
const parent = storage.nodes.get(this.parentId);
|
|
349
354
|
if (parent) {
|
|
350
|
-
|
|
355
|
+
return await parent.childExistsInSyncedVersion(this.objId);
|
|
351
356
|
}
|
|
352
357
|
}
|
|
353
|
-
return status;
|
|
354
358
|
}
|
|
355
359
|
async updateStatusInfo() {
|
|
356
360
|
const storage = this.syncedStorage();
|
|
357
361
|
const status = await storage.updateStatusInfo(this.objId);
|
|
362
|
+
status.existsInSyncedParent = await this.isThisPresentInSyncedParent();
|
|
358
363
|
return status;
|
|
359
364
|
}
|
|
360
365
|
isSyncedVersionOnDisk(version) {
|
|
@@ -137,9 +137,7 @@ class NodePersistance {
|
|
|
137
137
|
};
|
|
138
138
|
}
|
|
139
139
|
async writableSink(newVersion, attrs, xattrs, base) {
|
|
140
|
-
const segWriter = await (base ?
|
|
141
|
-
this.segWriterWithBase(newVersion, base) :
|
|
142
|
-
this.segWriter(newVersion));
|
|
140
|
+
const segWriter = await (base ? this.segWriterWithBase(newVersion, base) : this.segWriter(newVersion));
|
|
143
141
|
const { sink, sub } = (0, xsp_files_1.makeEncryptingByteSink)(segWriter);
|
|
144
142
|
const defSink = (0, deferred_1.defer)();
|
|
145
143
|
return {
|
|
@@ -605,8 +605,7 @@ class WritablePayloadV2 {
|
|
|
605
605
|
if (this.sectionsInSink.length === 0) {
|
|
606
606
|
return;
|
|
607
607
|
}
|
|
608
|
-
let fstNoCutSectionInd = this.sectionsInSink
|
|
609
|
-
.findIndex(s => (s.ofsInSrc >= this.cutonlyAreaEnd));
|
|
608
|
+
let fstNoCutSectionInd = this.sectionsInSink.findIndex(s => (s.ofsInSrc >= this.cutonlyAreaEnd));
|
|
610
609
|
if (fstNoCutSectionInd < 0) {
|
|
611
610
|
fstNoCutSectionInd = this.sectionsInSink.length;
|
|
612
611
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { SingleProc } from '../processes/synced';
|
|
2
|
+
type WritableFS = web3n.files.WritableFS;
|
|
3
|
+
/**
|
|
4
|
+
* Log entry in synced files that act as broadcast of what operation a particular device has done.
|
|
5
|
+
* There may be race condition, in which devices may perform and record same operation simultaneously in different
|
|
6
|
+
* log files.
|
|
7
|
+
* There will also be situations when operation is recorded only on one device, like when triggered by action on
|
|
8
|
+
* one device.
|
|
9
|
+
*
|
|
10
|
+
* Regular log entry has payload in `p` and `n` with entry number that is an increment counter within a given log.
|
|
11
|
+
* When data set is synced to server, under some version `v`, a log entry with `syncPoint` is added.
|
|
12
|
+
* `syncPoint` contains `logSyncedPoints` with last operations on known devices that have been incorporated into
|
|
13
|
+
* the data set synced state of given version.
|
|
14
|
+
*
|
|
15
|
+
* Sync point information allows to ignore log entries, cutting log files accordingly.
|
|
16
|
+
*/
|
|
17
|
+
export interface LogEntry<T> {
|
|
18
|
+
/**
|
|
19
|
+
* When data set is synced to server, under some version `v`, this log entry is added.
|
|
20
|
+
* This contains `logSyncedPoints` with last operations on known devices that have been incorporated into data
|
|
21
|
+
* set synced state.
|
|
22
|
+
*/
|
|
23
|
+
syncPoint?: {
|
|
24
|
+
v: number;
|
|
25
|
+
logSyncedPoints: {
|
|
26
|
+
[devId: string]: number;
|
|
27
|
+
};
|
|
28
|
+
params: any;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Increasing log entry number, unique to a given device log.
|
|
32
|
+
*/
|
|
33
|
+
n: number;
|
|
34
|
+
/**
|
|
35
|
+
* Timestamp of log helps to somewhat order logs from many device.
|
|
36
|
+
*/
|
|
37
|
+
ts: number;
|
|
38
|
+
/**
|
|
39
|
+
* Payload of regular log entry.
|
|
40
|
+
*/
|
|
41
|
+
p?: T;
|
|
42
|
+
}
|
|
43
|
+
export declare function makeRecentChangesLogs<T>(logsFS: WritableFS, localLogsFS: WritableFS, areOpsEqual: (a: T, b: T) => boolean, logError: (err: any, msg?: string) => Promise<void>): Promise<{
|
|
44
|
+
appendLogsAndUpload: (...ops: T[]) => Promise<void> | undefined;
|
|
45
|
+
watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, sink: (ops: LogEntry<T>[]) => Promise<void>) => () => void;
|
|
46
|
+
numOfLogs: () => number;
|
|
47
|
+
cutLogOnDatasetSyncAndUpload: (syncPointV: number, params?: any) => Promise<void> | undefined;
|
|
48
|
+
}>;
|
|
49
|
+
export type RecentChangesLogs<T> = Awaited<ReturnType<typeof makeRecentChangesLogs<T>>>;
|
|
50
|
+
export interface DatasetInstance<T> {
|
|
51
|
+
watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, resetSyncInterval: () => void) => (() => void);
|
|
52
|
+
absorbOpsFromOtherDevices: (changeProc: SingleProc, ops: LogEntry<T>[]) => Promise<void>;
|
|
53
|
+
}
|
|
54
|
+
export declare function startDatasetSync<T>(periodicCheckAndUploadOfData: () => Promise<void>, millisBeforeNextRun: () => number, logs: Pick<RecentChangesLogs<T>, 'watchAndApplyOpsFromOtherDevices'>, dataset: DatasetInstance<T>): {
|
|
55
|
+
changeProc: SingleProc;
|
|
56
|
+
stopSyncing: () => void;
|
|
57
|
+
};
|
|
58
|
+
export {};
|