core-3nweb-client-lib 0.49.6 → 0.49.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/api-defs/asmail.d.ts +1 -0
- package/build/api-defs/files.d.ts +11 -5
- package/build/core/asmail/delivery/common.d.ts +8 -28
- package/build/core/asmail/delivery/common.js +1 -47
- package/build/core/asmail/delivery/index.js +3 -3
- package/build/core/asmail/delivery/per-recipient-wip.js +3 -2
- package/build/core/asmail/inbox/index.d.ts +8 -32
- package/build/core/asmail/inbox/index.js +120 -170
- package/build/core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.d.ts +11 -0
- package/build/{lib-client/xsp-fs/util/for-arrays.js → core/asmail/inbox/msg-indexing/dataset/v0/sql-db-v-none.js} +7 -13
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.d.ts +33 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/logs.js +72 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.d.ts +28 -0
- package/build/core/asmail/inbox/msg-indexing/dataset/v1/sql-db.js +477 -0
- package/build/core/asmail/inbox/msg-indexing/index.d.ts +12 -26
- package/build/core/asmail/inbox/msg-indexing/index.js +47 -68
- package/build/core/asmail/inbox/msg-on-disk.d.ts +0 -5
- package/build/core/asmail/inbox/msg-on-disk.js +2 -24
- package/build/core/asmail/index.js +8 -7
- package/build/core/asmail/msg/common.d.ts +0 -1
- package/build/core/asmail/msg/common.js +1 -1
- package/build/core/index.js +7 -4
- package/build/core/keyring/common.d.ts +1 -0
- package/build/core/keyring/common.js +2 -1
- package/build/core/keyring/index.d.ts +7 -44
- package/build/core/keyring/index.js +88 -222
- package/build/core/keyring/introductory-keys/published-intro-key.d.ts +17 -0
- package/build/core/keyring/introductory-keys/published-intro-key.js +190 -0
- package/build/core/keyring/peer-keys/dataset/v1/json-files.d.ts +105 -0
- package/build/core/keyring/peer-keys/dataset/v1/json-files.js +89 -0
- package/build/core/keyring/peer-keys/dataset/v2/logs.d.ts +43 -0
- package/build/core/keyring/peer-keys/dataset/v2/logs.js +116 -0
- package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.d.ts +73 -0
- package/build/core/keyring/peer-keys/dataset/v2/peer-keys-db.js +543 -0
- package/build/core/keyring/peer-keys/index.d.ts +24 -0
- package/build/core/keyring/peer-keys/index.js +208 -0
- package/build/core/storage/synced/obj-files.js +10 -3
- package/build/core/storage/synced/obj-status.d.ts +1 -1
- package/build/core/storage/synced/obj-status.js +8 -2
- package/build/core/storage/synced/storage.d.ts +4 -4
- package/build/core/storage/system-folders/apps-data.d.ts +0 -1
- package/build/core/storage/system-folders/apps-data.js +2 -6
- package/build/lib-client/asmail/recipient.d.ts +1 -0
- package/build/lib-client/asmail/recipient.js +11 -0
- package/build/lib-client/fs-utils/fs-sync-utils.d.ts +1 -1
- package/build/lib-client/fs-utils/fs-sync-utils.js +70 -13
- package/build/lib-client/xsp-fs/folder-node.js +13 -3
- package/build/lib-client/xsp-fs/fs.js +3 -1
- package/build/lib-client/xsp-fs/node-in-fs.d.ts +1 -0
- package/build/lib-client/xsp-fs/node-in-fs.js +7 -2
- package/build/lib-client/xsp-fs/node-persistence.js +1 -3
- package/build/lib-client/xsp-fs/xsp-payload-v2.js +1 -2
- package/build/lib-common/dataset-sync/per-device-json-array-logs.d.ts +58 -0
- package/build/lib-common/dataset-sync/per-device-json-array-logs.js +432 -0
- package/build/lib-common/dataset-sync/single-file.d.ts +10 -0
- package/build/lib-common/dataset-sync/single-file.js +77 -0
- package/build/lib-common/number-line.d.ts +10 -0
- package/build/lib-common/number-line.js +66 -0
- package/build/lib-common/objs-on-disk/obj-version-file.d.ts +1 -1
- package/build/lib-common/objs-on-disk/obj-version-file.js +3 -3
- package/build/lib-common/rng-def.d.ts +1 -0
- package/build/lib-common/rng-def.js +8 -0
- package/build/lib-sqlite-on-3nstorage/for-sqlite.d.ts +38 -0
- package/build/lib-sqlite-on-3nstorage/for-sqlite.js +162 -0
- package/build/lib-sqlite-on-3nstorage/index.d.ts +18 -5
- package/build/lib-sqlite-on-3nstorage/index.js +49 -26
- package/build/tests/caps-api/file-sink-checks/splice.js +72 -1
- package/build/tests/caps-api/fs-checks/sync-on-one-dev/upload.js +27 -1
- package/build/tests/caps-api/fs-checks/sync-with-two-devs/update-propagation.js +12 -3
- package/build/tests/computer.3nweb.core/inbox/msg-indexing.js +16 -12
- package/build/tests/computer.3nweb.core/keyrings.js +113 -28
- package/build/tests/jasmine.js +2 -2
- package/build/tests/units/number-line.js +9 -9
- package/package.json +2 -1
- package/build/core/asmail/inbox/msg-indexing/logs-n-entries.d.ts +0 -52
- package/build/core/asmail/inbox/msg-indexing/logs-n-entries.js +0 -333
- package/build/core/asmail/inbox/msg-indexing/sql-indexing.d.ts +0 -4
- package/build/core/asmail/inbox/msg-indexing/sql-indexing.js +0 -319
- package/build/core/keyring/correspondent-keys.d.ts +0 -149
- package/build/core/keyring/correspondent-keys.js +0 -403
- package/build/core/keyring/id-to-email-map.d.ts +0 -38
- package/build/core/keyring/id-to-email-map.js +0 -105
- package/build/core/keyring/keyring-storage.d.ts +0 -9
- package/build/core/keyring/keyring-storage.js +0 -54
- package/build/core/keyring/published-intro-key.d.ts +0 -45
- package/build/core/keyring/published-intro-key.js +0 -200
- package/build/lib-client/xsp-fs/util/for-arrays.d.ts +0 -1
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { SingleProc } from '../processes/synced';
|
|
2
|
+
type WritableFS = web3n.files.WritableFS;
|
|
3
|
+
/**
|
|
4
|
+
* Log entry in synced files that act as broadcast of what operation a particular device has done.
|
|
5
|
+
* There may be race condition, in which devices may perform and record same operation simultaneously in different
|
|
6
|
+
* log files.
|
|
7
|
+
* There will also be situations when operation is recorded only on one device, like when triggered by action on
|
|
8
|
+
* one device.
|
|
9
|
+
*
|
|
10
|
+
* Regular log entry has payload in `p` and `n` with entry number that is an increment counter within a given log.
|
|
11
|
+
* When data set is synced to server, under some version `v`, a log entry with `syncPoint` is added.
|
|
12
|
+
* `syncPoint` contains `logSyncedPoints` with last operations on known devices that have been incorporated into
|
|
13
|
+
* the data set synced state of given version.
|
|
14
|
+
*
|
|
15
|
+
* Sync point information allows to ignore log entries, cutting log files accordingly.
|
|
16
|
+
*/
|
|
17
|
+
export interface LogEntry<T> {
|
|
18
|
+
/**
|
|
19
|
+
* When data set is synced to server, under some version `v`, this log entry is added.
|
|
20
|
+
* This contains `logSyncedPoints` with last operations on known devices that have been incorporated into data
|
|
21
|
+
* set synced state.
|
|
22
|
+
*/
|
|
23
|
+
syncPoint?: {
|
|
24
|
+
v: number;
|
|
25
|
+
logSyncedPoints: {
|
|
26
|
+
[devId: string]: number;
|
|
27
|
+
};
|
|
28
|
+
params: any;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Increasing log entry number, unique to a given device log.
|
|
32
|
+
*/
|
|
33
|
+
n: number;
|
|
34
|
+
/**
|
|
35
|
+
* Timestamp of log helps to somewhat order logs from many device.
|
|
36
|
+
*/
|
|
37
|
+
ts: number;
|
|
38
|
+
/**
|
|
39
|
+
* Payload of regular log entry.
|
|
40
|
+
*/
|
|
41
|
+
p?: T;
|
|
42
|
+
}
|
|
43
|
+
export declare function makeRecentChangesLogs<T>(logsFS: WritableFS, localLogsFS: WritableFS, areOpsEqual: (a: T, b: T) => boolean, logError: (err: any, msg?: string) => Promise<void>): Promise<{
|
|
44
|
+
appendLogsAndUpload: (...ops: T[]) => Promise<void> | undefined;
|
|
45
|
+
watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, sink: (ops: LogEntry<T>[]) => Promise<void>) => () => void;
|
|
46
|
+
numOfLogs: () => number;
|
|
47
|
+
cutLogOnDatasetSyncAndUpload: (syncPointV: number, params?: any) => Promise<void> | undefined;
|
|
48
|
+
}>;
|
|
49
|
+
export type RecentChangesLogs<T> = Awaited<ReturnType<typeof makeRecentChangesLogs<T>>>;
|
|
50
|
+
export interface DatasetInstance<T> {
|
|
51
|
+
watchAndApplyOpsFromOtherDevices: (changeProc: SingleProc, resetSyncInterval: () => void) => (() => void);
|
|
52
|
+
absorbOpsFromOtherDevices: (changeProc: SingleProc, ops: LogEntry<T>[]) => Promise<void>;
|
|
53
|
+
}
|
|
54
|
+
export declare function startDatasetSync<T>(periodicCheckAndUploadOfData: () => Promise<void>, millisBeforeNextRun: () => number, logs: Pick<RecentChangesLogs<T>, 'watchAndApplyOpsFromOtherDevices'>, dataset: DatasetInstance<T>): {
|
|
55
|
+
changeProc: SingleProc;
|
|
56
|
+
stopSyncing: () => void;
|
|
57
|
+
};
|
|
58
|
+
export {};
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2026 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.makeRecentChangesLogs = makeRecentChangesLogs;
|
|
20
|
+
exports.startDatasetSync = startDatasetSync;
|
|
21
|
+
const synced_1 = require("../processes/synced");
|
|
22
|
+
const buffer_utils_1 = require("../buffer-utils");
|
|
23
|
+
const rng_def_1 = require("../rng-def");
|
|
24
|
+
const sink_utils_1 = require("../obj-streaming/sink-utils");
|
|
25
|
+
const DEVICE_ID_FILE = 'device-id';
|
|
26
|
+
const DEV_LOGS_PROCESSING_FILE = 'dev-logs-processing';
|
|
27
|
+
const DEV_ID_LEN = 24;
|
|
28
|
+
const LOG_EXT = '.recent-changes';
|
|
29
|
+
function getOrMakeIdForThisDevice(localFS, logsFS) {
|
|
30
|
+
return localFS.readTxtFile(DEVICE_ID_FILE)
|
|
31
|
+
.catch(async (exc) => {
|
|
32
|
+
if (!exc.notFound) {
|
|
33
|
+
throw exc;
|
|
34
|
+
}
|
|
35
|
+
let devId = await (0, rng_def_1.stringOfB64UrlSafeChars)(DEV_ID_LEN, rng_def_1.pseudoRNG);
|
|
36
|
+
while (await logsFS.checkFilePresence(logsFileName(devId))) {
|
|
37
|
+
devId = await (0, rng_def_1.stringOfB64UrlSafeChars)(DEV_ID_LEN, rng_def_1.pseudoRNG);
|
|
38
|
+
}
|
|
39
|
+
await localFS.writeTxtFile(DEVICE_ID_FILE, devId);
|
|
40
|
+
return devId;
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
function logsFileName(devId) {
|
|
44
|
+
return `${devId}${LOG_EXT}`;
|
|
45
|
+
}
|
|
46
|
+
function devIdFromFileName(fName) {
|
|
47
|
+
if (fName.endsWith(LOG_EXT)) {
|
|
48
|
+
return fName.slice(0, fName.length - LOG_EXT.length);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function serializeLogs(logs) {
|
|
52
|
+
const json = JSON.stringify(logs);
|
|
53
|
+
return json.substring(1, json.length - 1);
|
|
54
|
+
}
|
|
55
|
+
function deserializeLogs(choppedJSON) {
|
|
56
|
+
const json = `[${choppedJSON}]`;
|
|
57
|
+
return JSON.parse(json);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* This is a catch callback, which returns undefined on file(folder) not found
|
|
61
|
+
* exception, and re-throws all other exceptions/errors.
|
|
62
|
+
*/
|
|
63
|
+
function notFoundOrReThrow(exc) {
|
|
64
|
+
if (!exc.notFound) {
|
|
65
|
+
throw exc;
|
|
66
|
+
}
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
async function makeRecentChangesLogs(logsFS, localLogsFS, areOpsEqual, logError) {
|
|
70
|
+
const thisDevId = await getOrMakeIdForThisDevice(localLogsFS, logsFS);
|
|
71
|
+
const { logs, thisDevLogsFile } = await (async function readOrCreateLogsFile() {
|
|
72
|
+
const thisDevLogsFile = await logsFS.writableFile(logsFileName(thisDevId), { create: false })
|
|
73
|
+
.catch(notFoundOrReThrow);
|
|
74
|
+
if (thisDevLogsFile) {
|
|
75
|
+
const logsTxt = await thisDevLogsFile.readTxt();
|
|
76
|
+
try {
|
|
77
|
+
const logs = deserializeLogs(logsTxt);
|
|
78
|
+
return { logs, thisDevLogsFile };
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
// if there was anything wrong, by ignoring, we'll effectively overwrite file
|
|
82
|
+
await logError(err, `Fail to parse current changes from log file ${logsFS.name}/${thisDevLogsFile.name}`);
|
|
83
|
+
return { logs: [], thisDevLogsFile };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
const thisDevLogsFile = await logsFS.writableFile(logsFileName(thisDevId));
|
|
88
|
+
await thisDevLogsFile.writeTxt('');
|
|
89
|
+
return { logs: [], thisDevLogsFile };
|
|
90
|
+
}
|
|
91
|
+
})();
|
|
92
|
+
let numOfEntriesOnLastSave = logs.length;
|
|
93
|
+
let needToSave = false;
|
|
94
|
+
let needToUpload = (await thisDevLogsFile.v.sync.status(true)).state !== 'synced';
|
|
95
|
+
const { snapshotOpsInfoForSync, shouldProcessOp, addToRecent, addDoneOnThisDeviceRecent } = await makeLogsProcessingState(localLogsFS, thisDevId, areOpsEqual);
|
|
96
|
+
await saveAndUploadIfNeeded();
|
|
97
|
+
function watchAndApplyOpsFromOtherDevices(changeProc, sink) {
|
|
98
|
+
const stopWatching = logsFS.watchTree('.', 1, {
|
|
99
|
+
next: fsEvent => changeProc.startOrChain(async () => {
|
|
100
|
+
if (fsEvent.path.endsWith(thisDevLogsFile.name)) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (fsEvent.type === 'remote-change') {
|
|
104
|
+
if (fsEvent.path === '.') {
|
|
105
|
+
await logsFS.v.sync.adoptRemote(fsEvent.path);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
await doOnNewRemoteVersionOfLog(fsEvent, sink);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
else if ((fsEvent.type === 'entry-addition') && (fsEvent.src === 'sync') && fsEvent.entry.isFile) {
|
|
112
|
+
await doOnAppearanceOfNewDeviceLog(fsEvent.entry.name, sink);
|
|
113
|
+
}
|
|
114
|
+
})
|
|
115
|
+
});
|
|
116
|
+
changeProc.startOrChain(() => getAndProcessOpsFromOtherDevicesLogs(sink));
|
|
117
|
+
return stopWatching;
|
|
118
|
+
}
|
|
119
|
+
async function doOnAppearanceOfNewDeviceLog(logFile, sink) {
|
|
120
|
+
const devId = devIdFromFileName(logFile);
|
|
121
|
+
if (!devId) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
try {
|
|
125
|
+
const content = (await logsFS.readBytes(logFile));
|
|
126
|
+
if (!content) {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const appendedEvents = deserializeLogs(buffer_utils_1.utf8.open(content));
|
|
130
|
+
processFromLogFile(devId, appendedEvents, sink);
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
if (err.type !== 'connect') {
|
|
134
|
+
logError(err);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
function processFromLogFile(devId, ops, sink) {
|
|
139
|
+
const newOps = ops.filter(op => shouldProcessOp(devId, op));
|
|
140
|
+
if (newOps.length > 0) {
|
|
141
|
+
const recordCompletion = addToRecent(devId, newOps);
|
|
142
|
+
sink(newOps).finally(recordCompletion);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async function doOnNewRemoteVersionOfLog({ path, newRemoteVersion }, sink) {
|
|
146
|
+
const devId = devIdFromFileName(path);
|
|
147
|
+
if (!devId) {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
const { size: initSize, version: initVersion } = await logsFS.v.stat(path);
|
|
152
|
+
if (initVersion === newRemoteVersion) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
await logsFS.v.sync.adoptRemote(path);
|
|
156
|
+
const { size: newSize } = await logsFS.v.stat(path);
|
|
157
|
+
let additionalBytes;
|
|
158
|
+
if (initSize < newSize) {
|
|
159
|
+
additionalBytes = (await logsFS.readBytes(path, (initSize > 0) ? initSize + 1 : undefined));
|
|
160
|
+
}
|
|
161
|
+
else if (initSize > newSize) {
|
|
162
|
+
additionalBytes = (await logsFS.readBytes(path));
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const appendedOps = deserializeLogs(buffer_utils_1.utf8.open(additionalBytes));
|
|
168
|
+
processFromLogFile(devId, appendedOps, sink);
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
if (err.type !== 'connect') {
|
|
172
|
+
logError(err);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
function numOfLogs() {
|
|
177
|
+
return logs.length;
|
|
178
|
+
}
|
|
179
|
+
function appendLogsAndUpload(...ops) {
|
|
180
|
+
if (ops.length > 0) {
|
|
181
|
+
const logEntries = [];
|
|
182
|
+
const ts = Date.now();
|
|
183
|
+
let n = ((logs.length === 0) ? 0 : logs[logs.length - 1].n);
|
|
184
|
+
for (let p of ops) {
|
|
185
|
+
n += 1;
|
|
186
|
+
logEntries.push({ n, ts, p });
|
|
187
|
+
}
|
|
188
|
+
logs.push(...logEntries);
|
|
189
|
+
addDoneOnThisDeviceRecent(logEntries);
|
|
190
|
+
needToSave = true;
|
|
191
|
+
return saveAndUploadIfNeeded();
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function cutLogOnDatasetSyncAndUpload(syncPointV, params) {
|
|
195
|
+
if (logs.length === 0) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const lastN = logs[logs.length - 1].n;
|
|
199
|
+
logs.splice(0, logs.length, {
|
|
200
|
+
n: lastN + 1,
|
|
201
|
+
ts: Date.now(),
|
|
202
|
+
syncPoint: {
|
|
203
|
+
v: syncPointV,
|
|
204
|
+
logSyncedPoints: snapshotOpsInfoForSync(lastN),
|
|
205
|
+
params
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
needToSave = true;
|
|
209
|
+
return saveAndUploadIfNeeded();
|
|
210
|
+
}
|
|
211
|
+
async function saveAndUploadIfNeeded() {
|
|
212
|
+
if (needToSave && (logs.length > numOfEntriesOnLastSave)) {
|
|
213
|
+
needToSave = false;
|
|
214
|
+
if (numOfEntriesOnLastSave > 0) {
|
|
215
|
+
const logsToAppend = logs.slice(numOfEntriesOnLastSave);
|
|
216
|
+
const logsTxt = serializeLogs(logsToAppend);
|
|
217
|
+
const sink = await thisDevLogsFile.getByteSink(false);
|
|
218
|
+
try {
|
|
219
|
+
await (0, sink_utils_1.appendFiniteSink)(sink, buffer_utils_1.utf8.pack(','), false);
|
|
220
|
+
await (0, sink_utils_1.appendFiniteSink)(sink, buffer_utils_1.utf8.pack(logsTxt), true);
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
await sink.done(err);
|
|
224
|
+
}
|
|
225
|
+
numOfEntriesOnLastSave += logsToAppend.length;
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
const logsTxt = serializeLogs(logs);
|
|
229
|
+
const numOfLogs = logs.length;
|
|
230
|
+
await thisDevLogsFile.writeTxt(logsTxt)
|
|
231
|
+
.catch(_exc => {
|
|
232
|
+
needToSave = true;
|
|
233
|
+
});
|
|
234
|
+
numOfEntriesOnLastSave += numOfLogs;
|
|
235
|
+
}
|
|
236
|
+
needToUpload = true;
|
|
237
|
+
}
|
|
238
|
+
if (needToUpload) {
|
|
239
|
+
needToUpload = false;
|
|
240
|
+
await thisDevLogsFile.v.sync.upload()
|
|
241
|
+
.catch(_exc => {
|
|
242
|
+
needToUpload = true;
|
|
243
|
+
});
|
|
244
|
+
const { existsInSyncedParent } = await thisDevLogsFile.v.sync.status(true);
|
|
245
|
+
if (!existsInSyncedParent) {
|
|
246
|
+
await syncLogsFS().catch((exc) => {
|
|
247
|
+
if (exc.type !== 'connect') {
|
|
248
|
+
logError(exc, `Fail to sync folder with messages index logs`);
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
async function getAndProcessOpsFromOtherDevicesLogs(sink) {
|
|
255
|
+
// XXX current code assumes that dataset was brought up from synced point, if any, or has always existed
|
|
256
|
+
try {
|
|
257
|
+
await syncLogsFS();
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
if (err.type === 'connect') {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
logError(err);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
const otherDevs = (await logsFS.listFolder(''))
|
|
268
|
+
.map(({ name }) => devIdFromFileName(name))
|
|
269
|
+
.filter(devId => !!devId && (devId !== thisDevId));
|
|
270
|
+
const opsFromAllDevices = [];
|
|
271
|
+
const recordCompletions = [];
|
|
272
|
+
for (const devId of otherDevs) {
|
|
273
|
+
try {
|
|
274
|
+
const logContent = await logsFS.readTxtFile(logsFileName(devId));
|
|
275
|
+
const devOps = deserializeLogs(logContent);
|
|
276
|
+
const newOps = devOps.filter(op => shouldProcessOp(devId, op));
|
|
277
|
+
if (newOps.length > 0) {
|
|
278
|
+
opsFromAllDevices.push(...newOps);
|
|
279
|
+
recordCompletions.push(addToRecent(devId, newOps));
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
catch (err) {
|
|
283
|
+
logError(err);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
if (opsFromAllDevices.length > 0) {
|
|
287
|
+
opsFromAllDevices.sort((a, b) => (a.ts - b.ts));
|
|
288
|
+
sink(opsFromAllDevices).finally(() => Promise.all(recordCompletions.map(c => c())));
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
async function syncLogsFS() {
|
|
292
|
+
const { state } = await logsFS.v.sync.status('');
|
|
293
|
+
if (state === 'behind') {
|
|
294
|
+
await logsFS.v.sync.absorbRemoteFolderChanges('');
|
|
295
|
+
}
|
|
296
|
+
else if (state === 'conflicting') {
|
|
297
|
+
await logsFS.v.sync.absorbRemoteFolderChanges('');
|
|
298
|
+
await logsFS.v.sync.upload('');
|
|
299
|
+
}
|
|
300
|
+
else if (state === 'unsynced') {
|
|
301
|
+
await logsFS.v.sync.upload('');
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return {
|
|
305
|
+
appendLogsAndUpload,
|
|
306
|
+
watchAndApplyOpsFromOtherDevices,
|
|
307
|
+
numOfLogs,
|
|
308
|
+
cutLogOnDatasetSyncAndUpload
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
async function makeLogsProcessingState(localLogsFS, thisDevId, areOpsEqual, maxRecentLen = 50) {
|
|
312
|
+
const procStateFile = await localLogsFS.writableFile(DEV_LOGS_PROCESSING_FILE);
|
|
313
|
+
const pState = (procStateFile.isNew ?
|
|
314
|
+
{ logProcessedPoints: {} } : await procStateFile.readJSON());
|
|
315
|
+
const pStateSavingProc = new synced_1.SingleProc();
|
|
316
|
+
let needsSaving = false;
|
|
317
|
+
async function savePState() {
|
|
318
|
+
needsSaving = true;
|
|
319
|
+
pStateSavingProc.startOrChain(async () => {
|
|
320
|
+
if (needsSaving) {
|
|
321
|
+
needsSaving = false;
|
|
322
|
+
await procStateFile.writeJSON(pState);
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
async function recordOpAsProcessed(devId, opN) {
|
|
327
|
+
let lastProcessed = pState.logProcessedPoints[devId];
|
|
328
|
+
if (lastProcessed === undefined) {
|
|
329
|
+
pState.logProcessedPoints[devId] = opN;
|
|
330
|
+
}
|
|
331
|
+
else if (lastProcessed >= opN) {
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
pState.logProcessedPoints[devId] = opN;
|
|
336
|
+
}
|
|
337
|
+
await savePState();
|
|
338
|
+
if (recentOps.length > maxRecentLen) {
|
|
339
|
+
recentOps.splice(0, recentOps.length - maxRecentLen);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
function snapshotOpsInfoForSync(lastN) {
|
|
343
|
+
const info = {};
|
|
344
|
+
for (const [devId, opN] of Object.entries(pState.logProcessedPoints)) {
|
|
345
|
+
info[devId] = opN;
|
|
346
|
+
}
|
|
347
|
+
info[thisDevId] = lastN;
|
|
348
|
+
return info;
|
|
349
|
+
}
|
|
350
|
+
function hasOpBeenProcessed(devId, opN) {
|
|
351
|
+
const lastProcessed = pState.logProcessedPoints[devId];
|
|
352
|
+
return ((lastProcessed === undefined) ? false : (lastProcessed >= opN));
|
|
353
|
+
}
|
|
354
|
+
// XXX should switch to keeping log entry and devId to trim recents when they get applied
|
|
355
|
+
// should trim be done within recordOpAsProcessed() ?
|
|
356
|
+
// And may be we should keep recent by ts, so as not to drop too recent, when there is a lot of them.
|
|
357
|
+
const recentOps = [];
|
|
358
|
+
const recentSyncPoint = undefined;
|
|
359
|
+
function shouldProcessOp(devId, op) {
|
|
360
|
+
if (hasOpBeenProcessed(devId, op.n)) {
|
|
361
|
+
return false;
|
|
362
|
+
}
|
|
363
|
+
if (op.p === undefined) {
|
|
364
|
+
if (op.syncPoint) {
|
|
365
|
+
if (pState.lastSyncV >= op.syncPoint.v) {
|
|
366
|
+
return false;
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
return (!recentSyncPoint || (recentSyncPoint.v < op.syncPoint.v));
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
else {
|
|
377
|
+
return !recentOps.find(ev => areOpsEqual(ev, op.p));
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
function addToRecent(devId, newOps) {
|
|
381
|
+
newOps.sort((a, b) => (a.n - b.n));
|
|
382
|
+
const lastOpN = newOps[newOps.length - 1].n;
|
|
383
|
+
const payloads = newOps.map(({ p }) => p).filter(p => (p !== undefined));
|
|
384
|
+
if (payloads.length > 0) {
|
|
385
|
+
recentOps.splice(recentOps.length, 0, ...payloads);
|
|
386
|
+
}
|
|
387
|
+
return () => recordOpAsProcessed(devId, lastOpN);
|
|
388
|
+
}
|
|
389
|
+
function addDoneOnThisDeviceRecent(done) {
|
|
390
|
+
const payloads = done.map(({ p }) => p).filter(p => (p !== undefined));
|
|
391
|
+
recentOps.splice(0, 0, ...payloads);
|
|
392
|
+
}
|
|
393
|
+
return {
|
|
394
|
+
snapshotOpsInfoForSync,
|
|
395
|
+
shouldProcessOp,
|
|
396
|
+
addToRecent,
|
|
397
|
+
addDoneOnThisDeviceRecent
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
function startDatasetSync(periodicCheckAndUploadOfData, millisBeforeNextRun, logs, dataset) {
|
|
401
|
+
// We order fn calls and data-sync operation within single process chain to ensure consistency.
|
|
402
|
+
// This is simple, and may be good enough.
|
|
403
|
+
// We pass this single process point out to wrap fn's with it, in an attempt to explicitly keeping it at the
|
|
404
|
+
// boundary of incoming operations, either from fn calls, or from handling changes passed via fs events.
|
|
405
|
+
const changeProc = new synced_1.SingleProc();
|
|
406
|
+
function datasetCheckAndSyncAction() {
|
|
407
|
+
changeProc.startOrChain(periodicCheckAndUploadOfData);
|
|
408
|
+
}
|
|
409
|
+
let interval = undefined;
|
|
410
|
+
function resetSyncInterval() {
|
|
411
|
+
if (interval !== undefined) {
|
|
412
|
+
clearInterval(interval);
|
|
413
|
+
}
|
|
414
|
+
interval = setInterval(datasetCheckAndSyncAction, millisBeforeNextRun()).unref();
|
|
415
|
+
}
|
|
416
|
+
resetSyncInterval();
|
|
417
|
+
// XXX we should watch for reconnect events and run similar to start checks of possibly missed ops
|
|
418
|
+
// watch and absorb ops, coming from logs of other devices
|
|
419
|
+
const stopLogsWatching = logs.watchAndApplyOpsFromOtherDevices(changeProc, ops => dataset.absorbOpsFromOtherDevices(changeProc, ops));
|
|
420
|
+
// watch and absorb ops in dataset, coming from places that dataset sets itself
|
|
421
|
+
const stopDatasetWatching = dataset.watchAndApplyOpsFromOtherDevices(changeProc, resetSyncInterval);
|
|
422
|
+
function stopSyncing() {
|
|
423
|
+
if (interval !== undefined) {
|
|
424
|
+
stopLogsWatching();
|
|
425
|
+
stopDatasetWatching();
|
|
426
|
+
clearInterval(interval);
|
|
427
|
+
interval = undefined;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return { changeProc, stopSyncing };
|
|
431
|
+
}
|
|
432
|
+
Object.freeze(exports);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { SingleProc } from "../processes/synced";
|
|
2
|
+
type WritableFS = web3n.files.WritableFS;
|
|
3
|
+
type WritableFile = web3n.files.WritableFile;
|
|
4
|
+
export declare function initSyncedFile(syncedFS: WritableFS, fName: string, initNewFile: (newFile: WritableFile) => Promise<void>): Promise<WritableFile>;
|
|
5
|
+
export declare function watchAndApplyChangesFromOtherDevices(file: WritableFile, onChangeFromOtherDevices: (changeProc: SingleProc, file: WritableFile) => Promise<void>, onConflictingChangeFromOtherDevices: (changeProc: SingleProc, file: WritableFile) => Promise<void>): {
|
|
6
|
+
changeProc: SingleProc;
|
|
7
|
+
stopFileWatching: () => void;
|
|
8
|
+
triggerUpload: () => void;
|
|
9
|
+
};
|
|
10
|
+
export {};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2026 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.initSyncedFile = initSyncedFile;
|
|
20
|
+
exports.watchAndApplyChangesFromOtherDevices = watchAndApplyChangesFromOtherDevices;
|
|
21
|
+
const synced_1 = require("../processes/synced");
|
|
22
|
+
async function initSyncedFile(syncedFS, fName, initNewFile) {
|
|
23
|
+
const file = await syncedFS.writableFile(fName);
|
|
24
|
+
if (file.isNew) {
|
|
25
|
+
await initNewFile(file);
|
|
26
|
+
await file.v.sync.upload();
|
|
27
|
+
await syncedFS.v.sync.upload('.');
|
|
28
|
+
}
|
|
29
|
+
// XXX add initial sync no-connection handling
|
|
30
|
+
return file;
|
|
31
|
+
}
|
|
32
|
+
function watchAndApplyChangesFromOtherDevices(file, onChangeFromOtherDevices, onConflictingChangeFromOtherDevices) {
|
|
33
|
+
const changeProc = new synced_1.SingleProc();
|
|
34
|
+
const stopFileWatching = file.watch({
|
|
35
|
+
next: ev => {
|
|
36
|
+
if (ev.type === 'remote-change') {
|
|
37
|
+
changeProc.startOrChain(async () => {
|
|
38
|
+
await file.v.sync.adoptRemote({ remoteVersion: ev.newRemoteVersion })
|
|
39
|
+
.catch((exc) => {
|
|
40
|
+
if (exc.type === 'connect') {
|
|
41
|
+
// XXX need to have watcher for connection to continue processing
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
else if ((ev.type === 'file-change') && (ev.src === 'sync')) {
|
|
47
|
+
onChangeFromOtherDevices(changeProc, file);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
function triggerUpload() {
|
|
52
|
+
changeProc.startOrChain(async () => file.v.sync.upload())
|
|
53
|
+
.catch((exc) => {
|
|
54
|
+
if (exc.type === 'connect') {
|
|
55
|
+
// XXX need to have watcher for connection to complete upload
|
|
56
|
+
// Should this whole try-to-connect method be generic to reuse it?
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
file.v.sync.status()
|
|
61
|
+
.then(async ({ state }) => {
|
|
62
|
+
if (state === 'behind') {
|
|
63
|
+
await changeProc.startOrChain(() => file.v.sync.adoptRemote());
|
|
64
|
+
}
|
|
65
|
+
else if (state === 'conflicting') {
|
|
66
|
+
await onConflictingChangeFromOtherDevices(changeProc, file);
|
|
67
|
+
}
|
|
68
|
+
else if (state === 'unsynced') {
|
|
69
|
+
triggerUpload();
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
.catch(exc => {
|
|
73
|
+
// XXX need to have watcher for connection to continue processing
|
|
74
|
+
});
|
|
75
|
+
return { changeProc, stopFileWatching, triggerUpload };
|
|
76
|
+
}
|
|
77
|
+
Object.freeze(exports);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This is a utility function that adds a number into given number line segments.
|
|
3
|
+
* Each segment is represented by an array, in which 0-th element is the smallest number in the segment,
|
|
4
|
+
* while 1-st element is the largest. Segments don't overlap and are ordered as they would on a number line:
|
|
5
|
+
* segment with smaller numbers go first.
|
|
6
|
+
* This returns true when segment line was updated, and false, otherwise (then number was in segments already).
|
|
7
|
+
* @param segments
|
|
8
|
+
* @param n
|
|
9
|
+
*/
|
|
10
|
+
export declare function addToNumberLineSegments(segments: number[][], n: number): boolean;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2016 - 2017, 2026 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.addToNumberLineSegments = addToNumberLineSegments;
|
|
20
|
+
/**
|
|
21
|
+
* This is a utility function that adds a number into given number line segments.
|
|
22
|
+
* Each segment is represented by an array, in which 0-th element is the smallest number in the segment,
|
|
23
|
+
* while 1-st element is the largest. Segments don't overlap and are ordered as they would on a number line:
|
|
24
|
+
* segment with smaller numbers go first.
|
|
25
|
+
* This returns true when segment line was updated, and false, otherwise (then number was in segments already).
|
|
26
|
+
* @param segments
|
|
27
|
+
* @param n
|
|
28
|
+
*/
|
|
29
|
+
function addToNumberLineSegments(segments, n) {
|
|
30
|
+
for (let i = (segments.length - 1); i >= 0; i -= 1) {
|
|
31
|
+
const [low, high] = segments[i];
|
|
32
|
+
if (high < n) {
|
|
33
|
+
if ((high + 1) >= n) {
|
|
34
|
+
// segment should be grown on a high side.
|
|
35
|
+
// But now bigger segment won't merge with higher segment, cause if
|
|
36
|
+
// it does, it would've merge from lower side of a higher segment.
|
|
37
|
+
segments[i][1] = n;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
// new segment should be added higher than this one
|
|
41
|
+
segments.splice(i + 1, 0, [n, n]);
|
|
42
|
+
}
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
if ((low <= n) && (n <= high)) {
|
|
46
|
+
// do nothing as number falls into current segment
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
if (low <= (n + 1)) {
|
|
50
|
+
// segment should be grown on a lower side
|
|
51
|
+
segments[i][0] = n;
|
|
52
|
+
if ((i - 1) >= 0) {
|
|
53
|
+
// bigger segment may overlap with a lower one
|
|
54
|
+
const lowerSeg = segments[i - 1];
|
|
55
|
+
if ((lowerSeg[1] + 1) >= n) {
|
|
56
|
+
lowerSeg[1] = high;
|
|
57
|
+
segments.splice(i, 1);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
segments.unshift([n, n]);
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
Object.freeze(exports);
|
|
@@ -8,7 +8,7 @@ export declare class ObjVersionFile {
|
|
|
8
8
|
private constructor();
|
|
9
9
|
static forExisting(path: string): Promise<ObjVersionFile>;
|
|
10
10
|
static createNew(path: string): Promise<ObjVersionFile>;
|
|
11
|
-
private
|
|
11
|
+
private creatingFileOnDisk;
|
|
12
12
|
moveFile(newPath: string, newHeader: Uint8Array | undefined): Promise<void>;
|
|
13
13
|
removeFile(): Promise<void>;
|
|
14
14
|
saveLayout(): Promise<void>;
|
|
@@ -43,11 +43,11 @@ class ObjVersionFile {
|
|
|
43
43
|
static async createNew(path) {
|
|
44
44
|
const layout = file_layout_1.ObjVersionBytesLayout.forNewFile();
|
|
45
45
|
const objFile = new ObjVersionFile(path, layout);
|
|
46
|
-
objFile.
|
|
46
|
+
await objFile.creatingFileOnDisk();
|
|
47
47
|
return objFile;
|
|
48
48
|
}
|
|
49
|
-
|
|
50
|
-
this.writeProc.addStarted(createNewV1File(this.path));
|
|
49
|
+
async creatingFileOnDisk() {
|
|
50
|
+
await this.writeProc.addStarted(createNewV1File(this.path));
|
|
51
51
|
}
|
|
52
52
|
moveFile(newPath, newHeader) {
|
|
53
53
|
return this.writeProc.startOrChain(async () => {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export type AsyncRNG = (n: number) => Promise<Uint8Array>;
|
|
2
2
|
export declare function stringOfB64UrlSafeChars(numOfChars: number, random: AsyncRNG): Promise<string>;
|
|
3
3
|
export declare function stringOfB64Chars(numOfChars: number, random: AsyncRNG): Promise<string>;
|
|
4
|
+
export declare function pseudoRNG(n: number): Promise<Uint8Array>;
|